diff options
Diffstat (limited to 'tools/perf/python')
39 files changed, 18886 insertions, 112 deletions
diff --git a/tools/perf/python/SchedGui.py b/tools/perf/python/SchedGui.py new file mode 100755 index 000000000000..fa0d010fe528 --- /dev/null +++ b/tools/perf/python/SchedGui.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +# SchedGui.py - Python extension for perf script, basic GUI code for +# traces drawing and overview. +# +# Copyright (C) 2010 by Frederic Weisbecker <fweisbec@gmail.com> +# +# Ported to modern directory structure. + +"""SchedGui.py - Python extension for perf script, basic GUI code for traces drawing and overview.""" +from __future__ import annotations + +import importlib +from typing import Any + +class _DummyWx: + """Dummy wx module fallback when wxPython is not installed.""" + Frame = object + + +try: + wx: Any = importlib.import_module("wx") + WX_AVAILABLE = True +except ImportError: + wx = _DummyWx + WX_AVAILABLE = False + + +class RootFrame(wx.Frame): + """Main window frame for scheduling trace visualization.""" + Y_OFFSET = 100 + RECT_HEIGHT = 100 + RECT_SPACE = 50 + EVENT_MARKING_WIDTH = 5 + + def __init__(self, sched_tracer, title, parent=None, win_id=-1): + if not WX_AVAILABLE: + raise ImportError("You need to install the wxpython lib for this script") + wx.Frame.__init__(self, parent, win_id, title) + + self.dc = None + (self.screen_width, self.screen_height) = wx.GetDisplaySize() + self.screen_width -= 10 + self.screen_height -= 10 + self.zoom = 0.5 + self.scroll_scale = 20 + self.sched_tracer = sched_tracer + self.sched_tracer.set_root_win(self) + (self.ts_start, self.ts_end) = sched_tracer.interval() + self.update_width_virtual() + self.nr_rects = sched_tracer.nr_rectangles() + 1 + self.height_virtual = RootFrame.Y_OFFSET + \ + (self.nr_rects * (RootFrame.RECT_HEIGHT + RootFrame.RECT_SPACE)) + + # whole window panel + self.panel = wx.Panel(self, size=(self.screen_width, self.screen_height)) + + # scrollable container + # Create SplitterWindow + self.splitter = wx.SplitterWindow(self.panel, style=wx.SP_3D) + + # scrollable container (Top) + self.scroll = wx.ScrolledWindow(self.splitter) + self.scroll.SetScrollbars(self.scroll_scale, self.scroll_scale, + int(self.width_virtual // self.scroll_scale), + int(self.height_virtual // self.scroll_scale)) + self.scroll.EnableScrolling(True, True) + self.scroll.SetFocus() + + # scrollable drawing area + self.scroll_panel = wx.Panel(self.scroll, + size=(self.screen_width - 15, self.screen_height // 2)) + self.scroll_panel.Bind(wx.EVT_PAINT, self.on_paint) + self.scroll_panel.Bind(wx.EVT_KEY_DOWN, self.on_key_press) + self.scroll_panel.Bind(wx.EVT_LEFT_DOWN, self.on_mouse_down) + self.scroll.Bind(wx.EVT_KEY_DOWN, self.on_key_press) + self.scroll.Bind(wx.EVT_LEFT_DOWN, self.on_mouse_down) + + self.scroll_panel.SetSize(int(self.width_virtual), int(self.height_virtual)) + + # Create a separate panel for text (Bottom) + self.text_panel = wx.Panel(self.splitter) + self.text_sizer = wx.BoxSizer(wx.VERTICAL) + self.txt = wx.TextCtrl(self.text_panel, -1, "Click a bar to see details", + style=wx.TE_MULTILINE) + self.text_sizer.Add(self.txt, 1, wx.EXPAND | wx.ALL, 5) + self.text_panel.SetSizer(self.text_sizer) + + # Split the window + self.splitter.SplitHorizontally(self.scroll, self.text_panel, (self.screen_height * 3) // 4) + + # Main sizer to layout splitter + self.main_sizer = wx.BoxSizer(wx.VERTICAL) + self.main_sizer.Add(self.splitter, 1, wx.EXPAND) + self.panel.SetSizer(self.main_sizer) + + self.scroll.Fit() + self.Fit() + + self.Show(True) + + def us_to_px(self, val): + """Convert microseconds to pixels.""" + return val / (10 ** 3) * self.zoom + + def px_to_us(self, val): + """Convert pixels to microseconds.""" + return (val / self.zoom) * (10 ** 3) + + def scroll_start(self): + """Get scroll start position in pixels.""" + (x, y) = self.scroll.GetViewStart() + return (x * self.scroll_scale, y * self.scroll_scale) + + def scroll_start_us(self): + """Get scroll start position in microseconds.""" + (x, _) = self.scroll_start() + return self.px_to_us(x) + + def paint_rectangle_zone(self, nr, color, top_color, start, end): + """Draw a rectangle zone for a CPU.""" + offset_px = self.us_to_px(start - self.ts_start) + width_px = self.us_to_px(end - start) + + offset_py = RootFrame.Y_OFFSET + (nr * (RootFrame.RECT_HEIGHT + RootFrame.RECT_SPACE)) + width_py = RootFrame.RECT_HEIGHT + + dc = self.dc + + if top_color is not None: + (r, g, b) = top_color + top_color = wx.Colour(r, g, b) + brush = wx.Brush(top_color, wx.SOLID) + dc.SetBrush(brush) + dc.DrawRectangle(int(offset_px), int(offset_py), + int(width_px), RootFrame.EVENT_MARKING_WIDTH) + width_py -= RootFrame.EVENT_MARKING_WIDTH + offset_py += RootFrame.EVENT_MARKING_WIDTH + + (r, g, b) = color + color = wx.Colour(r, g, b) + brush = wx.Brush(color, wx.SOLID) + dc.SetBrush(brush) + dc.DrawRectangle(int(offset_px), int(offset_py), int(width_px), int(width_py)) + + def update_rectangles(self, start, end): + """Update rectangles in the given time window.""" + start += self.ts_start + end += self.ts_start + self.sched_tracer.fill_zone(start, end) + + def on_paint(self, event): + """Handle paint event.""" + window = event.GetEventObject() + dc = wx.PaintDC(window) + + # Clear background to avoid ghosting + dc.SetBackground(wx.Brush(window.GetBackgroundColour())) + dc.Clear() + + self.dc = dc + try: + width = min(self.width_virtual, self.screen_width) + (x, _) = self.scroll_start() + start = self.px_to_us(x) + end = self.px_to_us(x + width) + self.update_rectangles(start, end) + + # Draw CPU labels at the left edge of the visible area + (x_scroll, _) = self.scroll_start() + for nr in range(self.nr_rects): + offset_py = RootFrame.Y_OFFSET + (nr * (RootFrame.RECT_HEIGHT + RootFrame.RECT_SPACE)) + dc.DrawText(f"CPU {nr}", x_scroll + 10, offset_py + 10) + finally: + self.dc = None + + def rect_from_ypixel(self, y): + y -= RootFrame.Y_OFFSET + rect = y // (RootFrame.RECT_HEIGHT + RootFrame.RECT_SPACE) + height = y % (RootFrame.RECT_HEIGHT + RootFrame.RECT_SPACE) + + if rect < 0 or rect > self.nr_rects - 1 or height > RootFrame.RECT_HEIGHT: + return -1 + + return rect + + def update_summary(self, txt): + self.txt.SetValue(txt) + self.text_panel.Layout() + self.splitter.Layout() + self.text_panel.Refresh() + + def on_mouse_down(self, event): + pos = event.GetPosition() + x, y = pos.x, pos.y + rect = self.rect_from_ypixel(y) + if rect == -1: + return + + t = self.px_to_us(x) + self.ts_start + + self.sched_tracer.mouse_down(rect, t) + + def update_width_virtual(self): + self.width_virtual = self.us_to_px(self.ts_end - self.ts_start) + + def __zoom(self, x): + self.update_width_virtual() + (xpos, ypos) = self.scroll.GetViewStart() + xpos = int(self.us_to_px(x) // self.scroll_scale) + self.scroll_panel.SetSize((int(self.width_virtual), int(self.height_virtual))) + self.scroll.SetScrollbars(self.scroll_scale, self.scroll_scale, + int(self.width_virtual // self.scroll_scale), + int(self.height_virtual // self.scroll_scale), + xpos, ypos) + self.Refresh() + + def zoom_in(self): + x = self.scroll_start_us() + self.zoom *= 2 + self.__zoom(x) + + def zoom_out(self): + x = self.scroll_start_us() + self.zoom /= 2 + self.__zoom(x) + + def on_key_press(self, event): + key = event.GetRawKeyCode() + if key == ord("+"): + self.zoom_in() + return + if key == ord("-"): + self.zoom_out() + return + + key = event.GetKeyCode() + (x, y) = self.scroll.GetViewStart() + if key == wx.WXK_RIGHT: + self.scroll.Scroll(x + 1, y) + elif key == wx.WXK_LEFT: + self.scroll.Scroll(x - 1, y) + elif key == wx.WXK_DOWN: + self.scroll.Scroll(x, y + 1) + elif key == wx.WXK_UP: + self.scroll.Scroll(x, y - 1) diff --git a/tools/perf/python/arm-cs-trace-disasm.py b/tools/perf/python/arm-cs-trace-disasm.py new file mode 100755 index 000000000000..1812ccaf4b20 --- /dev/null +++ b/tools/perf/python/arm-cs-trace-disasm.py @@ -0,0 +1,440 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +""" +arm-cs-trace-disasm.py: ARM CoreSight Trace Dump With Disassembler + +Author: Tor Jeremiassen <tor@ti.com> + Mathieu Poirier <mathieu.poirier@linaro.org> + Leo Yan <leo.yan@linaro.org> + Al Grant <Al.Grant@arm.com> + +Below are some example commands for using this script. +Note a --kcore recording is required for accurate decode +due to the alternatives patching mechanism. In addition to this, +source line info comes from Perf, and when using kcore there is +no debug info. The following lists the supported features in each mode: + ++-----------+-----------------+------------------+------------------+ +| Recording | Accurate decode | Source line dump | Disassembly dump | ++-----------+-----------------+------------------+------------------+ +| --kcore | yes | no | yes | +| normal | no | yes (inaccurate) | yes (inaccurate) | ++-----------+-----------------+------------------+------------------+ + +Output disassembly with objdump and auto detect vmlinux +(when running on same machine): + perf script arm-cs-trace-disasm -- -d + +Output disassembly with llvm-objdump: + perf script arm-cs-trace-disasm -- -d llvm-objdump-11 -k path/to/vmlinux + +Output accurate disassembly by passing kcore to script: + perf script arm-cs-trace-disasm -- -d -k perf.data/kcore_dir/kcore + +Output only source line and symbols: + perf script arm-cs-trace-disasm +""" +from __future__ import annotations + +import os +from os import path +import re +from subprocess import CalledProcessError, check_output +import argparse +import platform +import sys +from typing import Dict, List, Optional + +import perf + +# Initialize global dicts and regular expression +DISASM_CACHE: Dict[str, List[str]] = {} +DISASM_RE = re.compile(r"^\s*([0-9a-fA-F]+):") +DISASM_FUNC_RE = re.compile(r"^\s*([0-9a-fA-F]+)\s.*:") +CACHE_SIZE = 1024 +class _State: + sample_idx: int = -1 + source_file_name: Optional[str] = None + line_number: Optional[int] = None + dso: Optional[str] = None + +_STATE = _State() + +KVER = platform.release() +VMLINUX_PATHS = [ + f"/usr/lib/debug/boot/vmlinux-{KVER}.debug", + f"/usr/lib/debug/lib/modules/{KVER}/vmlinux", + f"/lib/modules/{KVER}/build/vmlinux", + f"/usr/lib/debug/boot/vmlinux-{KVER}", + f"/boot/vmlinux-{KVER}", + "/boot/vmlinux", + "vmlinux", +] + +def default_objdump() -> str: + """Return the default objdump path from perf config or 'objdump'.""" + try: + config = perf.config_get("annotate.objdump") + return str(config) if config else "objdump" + except (AttributeError, TypeError): + return "objdump" + +def find_vmlinux() -> Optional[str]: + """Find the vmlinux file in standard paths.""" + if hasattr(find_vmlinux, "path"): + return getattr(find_vmlinux, "path") + + for v in VMLINUX_PATHS: + if os.access(v, os.R_OK): + setattr(find_vmlinux, "path", v) + return v + setattr(find_vmlinux, "path", None) + return None + +def get_dso_file_path(dso_name: str, dso_build_id: str, vmlinux: Optional[str]) -> str: + """Return the path to the DSO file.""" + # Locate DSO binaries in the perf build-id cache (~/.debug or PERF_BUILDID_DIR). + # Perf stores cached binaries under both <buildid_dir>/<dso_long_name>/<build_id>/elf + # and the canonical symlink/file layout <buildid_dir>/.build-id/<bid[:2]>/<bid[2:]>/elf. + buildid_dir = os.environ.get('PERF_BUILDID_DIR') + if not buildid_dir: + buildid_dir = os.path.join(os.environ.get('HOME', ''), '.debug') + + if (dso_name in ("[kernel.kallsyms]", "vmlinux", "kcore") + or dso_name.endswith("/vmlinux") or dso_name.endswith("/kcore")): + if vmlinux: + return vmlinux + if dso_build_id and dso_build_id != "[unknown]" and len(dso_build_id) > 2: + bid_candidate = os.path.join( + buildid_dir, ".build-id", dso_build_id[:2], dso_build_id[2:], "elf" + ) + if os.access(bid_candidate, os.R_OK): + return bid_candidate + for kname in (dso_name, "vmlinux", "[kernel.kallsyms]"): + candidate = os.path.join(buildid_dir, kname.lstrip("/"), dso_build_id, "elf") + if os.access(candidate, os.R_OK): + return candidate + return find_vmlinux() or dso_name + + if dso_name == "[vdso]": + append = "/vdso" + else: + append = "/elf" + + if dso_build_id and dso_build_id != "[unknown]" and len(dso_build_id) > 2: + bid_path = (buildid_dir + "/.build-id/" + dso_build_id[:2] + "/" + + dso_build_id[2:] + append).replace('//', '/') + if os.access(bid_path, os.R_OK): + return bid_path + + dso_path = buildid_dir + "/" + dso_name + "/" + dso_build_id + append + # Replace duplicate slash chars to single slash char + dso_path = dso_path.replace('//', '/') + return dso_path + +def read_disam(dso_fname: str, dso_start: int, start_addr: int, + stop_addr: int, objdump: str) -> List[str]: + """Read disassembly from a DSO file using objdump.""" + if stop_addr <= start_addr or (stop_addr - start_addr) > 0x100000: + return [] + addr_range = f"{start_addr}:{stop_addr}:{dso_start}:{dso_fname}" + + # Don't let the cache get too big, clear it when it hits max size + if len(DISASM_CACHE) > CACHE_SIZE: + DISASM_CACHE.clear() + + if addr_range in DISASM_CACHE: + disasm_output = DISASM_CACHE[addr_range] + else: + start_addr = start_addr - dso_start + stop_addr = stop_addr - dso_start + disasm = [objdump, "-d", "-z", + f"--start-address={start_addr:#x}", + f"--stop-address={stop_addr:#x}"] + disasm += [dso_fname] + try: + disasm_output = check_output(disasm).decode('utf-8', errors='replace').split('\n') + except (CalledProcessError, OSError): + return [] + if len(disasm_output) <= 512: + DISASM_CACHE[addr_range] = disasm_output + + return disasm_output + +def print_disam(dso_fname: str, dso_start: int, start_addr: int, + stop_addr: int, objdump: str) -> None: + """Print disassembly for a given address range.""" + for line in read_disam(dso_fname, dso_start, start_addr, stop_addr, objdump): + m = DISASM_FUNC_RE.search(line) + if m is None: + m = DISASM_RE.search(line) + if m is None: + continue + print(f"\t{line}") + +def print_sample(sample: perf.sample_event) -> None: + """Print sample details.""" + print(f"Sample = {{ cpu: {sample.sample_cpu:04d} addr: {sample.sample_addr:016x} " + f"phys_addr: {sample.sample_phys_addr:016x} ip: {sample.sample_ip:016x} " + f"pid: {sample.sample_pid} tid: {sample.sample_tid} period: {sample.sample_period} " + f"time: {sample.sample_time} index: {_STATE.sample_idx}}}") + +def common_start_str(comm: str, sample: perf.sample_event) -> str: + """Return common start string for sample output.""" + sec = int(sample.sample_time / 1000000000) + ns = sample.sample_time % 1000000000 + cpu = sample.sample_cpu + pid = sample.sample_pid + tid = sample.sample_tid + return f"{comm:>16s} {pid:5d}/{tid:<5d} [{cpu:04d}] {sec:9d}.{ns:09d} " + +def print_srccode(comm: str, sample: perf.sample_event, symbol: str, dso: str) -> None: + """Print source code and symbols for a sample.""" + ip = sample.sample_ip + if symbol == "[unknown]": + start_str = common_start_str(comm, sample) + f"{ip:x}".rjust(16).ljust(40) + else: + symoff = 0 + symoff = getattr(sample, 'sym_offset', 0) or 0 + offs = f"+{symoff:#x}" if symoff != 0 else "" + start_str = common_start_str(comm, sample) + (symbol + offs).ljust(40) + + source_file_name, line_number, source_line = sample.srccode() or (None, 0, None) + if source_file_name: + if _STATE.line_number == line_number and _STATE.source_file_name == source_file_name: + src_str = "" + else: + if len(source_file_name) > 40: + src_file = f"...{source_file_name[-37:]} " + else: + src_file = source_file_name.ljust(41) + + if source_line is None: + src_str = f"{src_file}{line_number:>4d} <source not found>" + else: + src_str = f"{src_file}{line_number:>4d} {source_line}" + _STATE.dso = None + elif dso == _STATE.dso: + src_str = "" + else: + src_str = dso + _STATE.dso = dso + + _STATE.line_number = line_number + _STATE.source_file_name = source_file_name + + print(start_str, src_str) + +class TraceDisasm: + """Class to handle trace disassembly.""" + def __init__(self, cli_options: argparse.Namespace): + self.options = cli_options + self.sample_idx = -1 + self.cpu_data: Dict[str, int] = {} + self.session: Optional[perf.session] = None + + def process_event(self, sample: perf.sample_event) -> None: + """Process a single perf event.""" + self.sample_idx += 1 + _STATE.sample_idx = self.sample_idx + + if self.options.start_time is not None and sample.sample_time < self.options.start_time: + return + if self.options.stop_time is not None and sample.sample_time > self.options.stop_time: + sys.exit(0) + if self.options.start_sample is not None and self.sample_idx < self.options.start_sample: + return + if self.options.stop_sample is not None and self.sample_idx > self.options.stop_sample: + sys.exit(0) + + ev_name = str(sample.evsel) + if self.options.verbose: + print(f"Event type: {ev_name}") + print_sample(sample) + + # Initialize CPU data if it's empty, and directly return back + # if this is the first tracing event for this CPU. This must happen + # before the dso == '[unknown]' check because the initial + # CS_ETM_TRACE_ON sample has ip == 0 (dso == '[unknown]') and + # carries the starting target address in sample.sample_addr. + if self.cpu_data.get(f"{sample.sample_cpu}addr") is None: + self.cpu_data[f"{sample.sample_cpu}addr"] = sample.sample_addr + return + + dso = getattr(sample, 'dso_long_name', None) or sample.dso or '[unknown]' + symbol = sample.symbol or '[unknown]' + dso_bid = sample.dso_bid or '[unknown]' + dso_start = sample.map_start + dso_end = sample.map_end + map_pgoff = sample.map_pgoff or 0 + + comm = "[unknown]" + try: + if self.session: + thread_info = self.session.find_thread(sample.sample_pid, sample.sample_tid) + if thread_info: + comm = thread_info.comm() or "[unknown]" + except (TypeError, AttributeError): + pass + + if dso == '[unknown]': + return + + if dso_start is None or dso_end is None: + print(f"Failed to find valid dso map for dso {dso}") + return + + if "instructions" in ev_name: + print_srccode(comm, sample, symbol, dso) + return + + if "branches" not in ev_name: + return + + self._process_branch(sample, comm, symbol, dso, dso_bid, dso_start, dso_end, map_pgoff) + + def _process_branch(self, sample: perf.sample_event, comm: str, symbol: str, dso: str, + dso_bid: str, dso_start: int, dso_end: int, map_pgoff: int) -> None: + """Helper to process branch events.""" + cpu = sample.sample_cpu + ip = sample.sample_ip + addr = sample.sample_addr + + # The format for packet is: + # + # +------------+------------+------------+ + # sample_prev: | addr | ip | cpu | + # +------------+------------+------------+ + # sample_next: | addr | ip | cpu | + # +------------+------------+------------+ + # + # Combine the two continuous packets to get the instruction range for + # sample_prev::cpu: + # + # [ sample_prev::addr .. sample_next::ip ] + # + # sample_prev::addr is stored into cpu_data and read back as + # 'start_addr' when the new packet arrives, and sample_next::ip + 4 is + # used for 'stop_addr' so objdump includes the branch instruction at + # sample_next::ip. + start_addr = self.cpu_data[str(cpu) + 'addr'] + stop_addr = ip + 4 + + # Record for previous sample packet + self.cpu_data[str(cpu) + 'addr'] = addr + + # Filter out zero start_address. Optionally identify CS_ETM_TRACE_ON packet + if start_addr == 0: + if stop_addr == 4 and self.options.verbose: + print(f"CPU{cpu}: CS_ETM_TRACE_ON packet is inserted") + return + + if start_addr < dso_start or start_addr > dso_end: + print(f"Start address {start_addr:#x} is out of range [ {dso_start:#x} .. " + f"{dso_end:#x} ] for dso {dso}") + return + + if stop_addr < dso_start or stop_addr > dso_end: + print(f"Stop address {stop_addr:#x} is out of range [ {dso_start:#x} .. " + f"{dso_end:#x} ] for dso {dso}") + return + + if self.options.objdump is not None: + # Kernel images (vmlinux / kcore) and non-PIE executables linked at + # the default 0x400000 base already have their virtual load addresses + # baked into the ELF symbol/section headers read by objdump, so + # dso_vm_start and map_pgoff_local must be 0 to avoid subtracting + # map_start from virtual sample addresses. + if (dso in ("[kernel.kallsyms]", "vmlinux", "kcore") + or dso.endswith("/vmlinux") or dso.endswith("/kcore")): + dso_vm_start = 0 + map_pgoff_local = 0 + elif dso_start == 0x400000: + dso_vm_start = 0 + map_pgoff_local = 0 + else: + dso_vm_start = dso_start + map_pgoff_local = map_pgoff + + dso_fname = get_dso_file_path(dso, dso_bid, self.options.vmlinux) + if path.exists(dso_fname): + print_disam(dso_fname, dso_vm_start, start_addr + map_pgoff_local, + stop_addr + map_pgoff_local, self.options.objdump) + else: + print(f"Failed to find dso {dso} for address range [ " + f"{start_addr + map_pgoff_local:#x} .. {stop_addr + map_pgoff_local:#x} ]") + + print_srccode(comm, sample, symbol, dso) + + def run(self) -> None: + """Run the trace disassembly session.""" + input_file = self.options.input or "perf.data" + if input_file != "-" and not os.path.exists(input_file): + print(f"Error: {input_file} not found.", file=sys.stderr) + sys.exit(1) + + print('ARM CoreSight Trace Data Assembler Dump') + try: + session_kwargs = { + "sample": self.process_event, + "itrace": self.options.itrace, + } + if self.options.vmlinux: + session_kwargs["vmlinux"] = self.options.vmlinux + try: + self.session = perf.session(perf.data(input_file), **session_kwargs) + except TypeError: + session_kwargs.pop("vmlinux", None) + self.session = perf.session(perf.data(input_file), **session_kwargs) + except (OSError, ValueError, KeyError, RuntimeError, TypeError, AttributeError) as e: + print(f"Error opening session: {e}", file=sys.stderr) + sys.exit(1) + + try: + self.session.process_events() + except (KeyboardInterrupt, SystemExit): + pass + finally: + # Break the reference cycle between self.session and the bound + # self.process_event callback so the C perf.session object is freed. + self.session = None + print('End') + +if __name__ == "__main__": + def int_arg(v: str) -> int: + """Helper for integer command line arguments.""" + val = int(v) + if val < 0: + raise argparse.ArgumentTypeError("Argument must be a positive integer") + return val + + arg_parser = argparse.ArgumentParser(description="ARM CoreSight Trace Dump With Disassembler") + arg_parser.add_argument("-i", "--input", help="input perf.data file") + arg_parser.add_argument("-k", "--vmlinux", default=os.environ.get("PERF_SYMBOL_VMLINUX"), + help="Set path to vmlinux file. Omit to autodetect") + arg_parser.add_argument("-d", "--objdump", nargs="?", const=default_objdump(), + help="Show disassembly. Can also be used to change the objdump path") + arg_parser.add_argument("-v", "--verbose", action="store_true", help="Enable debugging log") + arg_parser.add_argument("--start-time", type=int_arg, + help="Monotonic clock time of sample to start from.") + arg_parser.add_argument("--stop-time", type=int_arg, + help="Monotonic clock time of sample to stop at.") + arg_parser.add_argument("--itrace", default=os.environ.get("PERF_ITRACE") or "b", + help="Instruction tracing options.") + arg_parser.add_argument("--start-sample", type=int_arg, + help="Index of sample to start from.") + arg_parser.add_argument("--stop-sample", type=int_arg, + help="Index of sample to stop at.") + + parsed_options = arg_parser.parse_args() + if (parsed_options.start_time is not None and parsed_options.stop_time is not None and + parsed_options.start_time >= parsed_options.stop_time): + print("--start-time must less than --stop-time") + sys.exit(2) + if (parsed_options.start_sample is not None and parsed_options.stop_sample is not None and + parsed_options.start_sample >= parsed_options.stop_sample): + print("--start-sample must less than --stop-sample") + sys.exit(2) + + td = TraceDisasm(parsed_options) + td.run() diff --git a/tools/perf/python/check-perf-trace.py b/tools/perf/python/check-perf-trace.py new file mode 100755 index 000000000000..a7679a897cc7 --- /dev/null +++ b/tools/perf/python/check-perf-trace.py @@ -0,0 +1,223 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +""" +Basic test of Python scripting support for perf. +Ported from tools/perf/scripts/python/check-perf-trace.py +""" +from __future__ import annotations + +import argparse +import collections +import perf + +unhandled: collections.defaultdict[str, int] = collections.defaultdict(int) +session = None + +softirq_vecs = { + 0: "HI_SOFTIRQ", + 1: "TIMER_SOFTIRQ", + 2: "NET_TX_SOFTIRQ", + 3: "NET_RX_SOFTIRQ", + 4: "BLOCK_SOFTIRQ", + 5: "IRQ_POLL_SOFTIRQ", + 6: "TASKLET_SOFTIRQ", + 7: "SCHED_SOFTIRQ", + 8: "HRTIMER_SOFTIRQ", + 9: "RCU_SOFTIRQ", +} + +_GFP_DMA = 1 << 0 +_GFP_HIGHMEM = 1 << 1 +_GFP_DMA32 = 1 << 2 +_GFP_MOVABLE = 1 << 3 +_GFP_RECLAIMABLE = 1 << 4 +_GFP_HIGH = 1 << 5 +_GFP_IO = 1 << 6 +_GFP_FS = 1 << 7 +_GFP_ZERO = 1 << 8 +_GFP_DIRECT_RECLAIM = 1 << 10 +_GFP_KSWAPD_RECLAIM = 1 << 11 +_GFP_WRITE = 1 << 12 +_GFP_NOWARN = 1 << 13 +_GFP_RETRY_MAYFAIL = 1 << 14 +_GFP_NOFAIL = 1 << 15 +_GFP_NORETRY = 1 << 16 +_GFP_MEMALLOC = 1 << 17 +_GFP_COMP = 1 << 18 +_GFP_NOMEMALLOC = 1 << 19 +_GFP_HARDWALL = 1 << 20 +_GFP_THISNODE = 1 << 21 +_GFP_ACCOUNT = 1 << 22 +_GFP_ZEROTAGS = 1 << 23 + +_GFP_RECLAIM = _GFP_DIRECT_RECLAIM | _GFP_KSWAPD_RECLAIM +_GFP_KERNEL = _GFP_RECLAIM | _GFP_IO | _GFP_FS +_GFP_USER = _GFP_KERNEL | _GFP_HARDWALL +_GFP_HIGHUSER = _GFP_USER | _GFP_HIGHMEM +_GFP_HIGHUSER_MOVABLE = _GFP_HIGHUSER | _GFP_MOVABLE +_GFP_TRANSHUGE_LIGHT = ( + _GFP_HIGHUSER_MOVABLE | _GFP_COMP | _GFP_NOMEMALLOC | _GFP_NOWARN +) & ~_GFP_RECLAIM +_GFP_TRANSHUGE = _GFP_TRANSHUGE_LIGHT | _GFP_DIRECT_RECLAIM + +GFP_FLAG_NAMES = [ + (_GFP_TRANSHUGE, "GFP_TRANSHUGE"), + (_GFP_TRANSHUGE_LIGHT, "GFP_TRANSHUGE_LIGHT"), + (_GFP_HIGHUSER_MOVABLE, "GFP_HIGHUSER_MOVABLE"), + (_GFP_HIGHUSER, "GFP_HIGHUSER"), + (_GFP_USER, "GFP_USER"), + (_GFP_KERNEL | _GFP_ACCOUNT, "GFP_KERNEL_ACCOUNT"), + (_GFP_KERNEL, "GFP_KERNEL"), + (_GFP_RECLAIM | _GFP_IO, "GFP_NOFS"), + (_GFP_HIGH | _GFP_KSWAPD_RECLAIM, "GFP_ATOMIC"), + (_GFP_RECLAIM, "GFP_NOIO"), + (_GFP_KSWAPD_RECLAIM | _GFP_NOWARN, "GFP_NOWAIT"), + (_GFP_DMA, "GFP_DMA"), + (_GFP_DMA32, "GFP_DMA32"), + (_GFP_RECLAIM, "__GFP_RECLAIM"), + (_GFP_DMA, "__GFP_DMA"), + (_GFP_HIGHMEM, "__GFP_HIGHMEM"), + (_GFP_DMA32, "__GFP_DMA32"), + (_GFP_MOVABLE, "__GFP_MOVABLE"), + (_GFP_RECLAIMABLE, "__GFP_RECLAIMABLE"), + (_GFP_HIGH, "__GFP_HIGH"), + (_GFP_IO, "__GFP_IO"), + (_GFP_FS, "__GFP_FS"), + (_GFP_ZERO, "__GFP_ZERO"), + (_GFP_DIRECT_RECLAIM, "__GFP_DIRECT_RECLAIM"), + (_GFP_KSWAPD_RECLAIM, "__GFP_KSWAPD_RECLAIM"), + (_GFP_WRITE, "__GFP_WRITE"), + (_GFP_NOWARN, "__GFP_NOWARN"), + (_GFP_RETRY_MAYFAIL, "__GFP_RETRY_MAYFAIL"), + (_GFP_NOFAIL, "__GFP_NOFAIL"), + (_GFP_NORETRY, "__GFP_NORETRY"), + (_GFP_MEMALLOC, "__GFP_MEMALLOC"), + (_GFP_COMP, "__GFP_COMP"), + (_GFP_NOMEMALLOC, "__GFP_NOMEMALLOC"), + (_GFP_HARDWALL, "__GFP_HARDWALL"), + (_GFP_THISNODE, "__GFP_THISNODE"), + (_GFP_ACCOUNT, "__GFP_ACCOUNT"), + (_GFP_ZEROTAGS, "__GFP_ZEROTAGS"), +] + + +def trace_begin() -> None: + """Called at the start of trace processing.""" + print("in trace_begin") + +def trace_end() -> None: + """Called at the end of trace processing.""" + print_unhandled() + print("in trace_end") + +def symbol_str(event_name: str, field_name: str, value: int) -> str: + """Resolves symbol values to strings.""" + # Note: The standalone Python API currently lacks dynamic libtraceevent + # formatting (equivalent to _perf_trace_context.symbol_str()) + if event_name == "irq__softirq_entry" and field_name == "vec": + return softirq_vecs.get(value, str(value)) + return str(value) + +def flag_str(event_name: str, field_name: str, value: int) -> str: + """Resolves flag values to strings.""" + # Note: The standalone Python API currently lacks dynamic libtraceevent + # formatting (equivalent to _perf_trace_context.flag_str()) + if event_name == "kmem__kmalloc" and field_name == "gfp_flags": + if value == 0: + return "none" + names = [] + rem = value + for mask, name in GFP_FLAG_NAMES: + if (rem & mask) == mask: + names.append(name) + rem &= ~mask + if rem: + names.append(f"0x{rem:x}") + return "|".join(names) + return str(value) + +def print_header(event_name: str, sample: perf.sample_event) -> None: + """Prints common header for events.""" + secs = sample.sample_time // 1000000000 + nsecs = sample.sample_time % 1000000000 + comm = "[unknown]" + try: + if session: + thread = session.find_thread(sample.sample_pid, sample.sample_tid) + if thread: + comm = thread.comm() or "[unknown]" + except (TypeError, AttributeError): + pass + print(f"{event_name:<20} {sample.sample_cpu:5} {secs:05}.{nsecs:09} " + f"{sample.sample_tid:8} {comm:<20} ", end=' ') + +def print_uncommon(sample: perf.sample_event) -> None: + """Prints uncommon fields for tracepoints.""" + # Fallback to 0 if field not found (e.g. on older kernels or if not tracepoint) + pc = getattr(sample, 'common_preempt_count', 0) + flags = getattr(sample, 'common_flags', 0) + lock_depth = getattr(sample, 'common_lock_depth', 0) + + print(f"common_preempt_count={pc}, common_flags={flags}, " + f"common_lock_depth={lock_depth}, ", end='') + +def irq__softirq_entry(sample: perf.sample_event) -> None: + """Handles irq:softirq_entry events.""" + print_header("irq__softirq_entry", sample) + print_uncommon(sample) + print(f"vec={symbol_str('irq__softirq_entry', 'vec', getattr(sample, 'vec', 0))}") + +def kmem__kmalloc(sample: perf.sample_event) -> None: + """Handles kmem:kmalloc events.""" + print_header("kmem__kmalloc", sample) + print_uncommon(sample) + + print(f"call_site={getattr(sample, 'call_site', 0):#x}, " + f"ptr={getattr(sample, 'ptr', 0):#x}, " + f"bytes_req={getattr(sample, 'bytes_req', 0):d}, " + f"bytes_alloc={getattr(sample, 'bytes_alloc', 0):d}, " + f"gfp_flags={flag_str('kmem__kmalloc', 'gfp_flags', getattr(sample, 'gfp_flags', 0))}") + +def trace_unhandled(event_name: str) -> None: + """Tracks unhandled events.""" + unhandled[event_name] += 1 + +def print_unhandled() -> None: + """Prints summary of unhandled events.""" + if not unhandled: + return + print("\nunhandled events:\n") + print(f"{'event':<40} {'count':>10}") + print("---------------------------------------- -----------") + for event_name, count in unhandled.items(): + print(f"{event_name:<40} {count:10}") + +def process_event(sample: perf.sample_event) -> None: + """Callback for processing events.""" + event_name = str(sample.evsel) + if event_name.startswith("evsel(") and event_name.endswith(")"): + event_name = event_name[6:-1] + if event_name.startswith("irq:softirq_entry"): + irq__softirq_entry(sample) + elif event_name.startswith("kmem:kmalloc"): + kmem__kmalloc(sample) + else: + trace_unhandled(event_name) + +if __name__ == "__main__": + ap = argparse.ArgumentParser() + ap.add_argument("-i", "--input", default="perf.data", help="Input file name") + args = ap.parse_args() + + trace_begin() + try: + session = perf.session(perf.data(args.input), sample=process_event) + try: + session.process_events() + except KeyboardInterrupt: + pass + trace_end() + finally: + # Break the reference cycle between the global session and the + # process_event callback so the C perf.session object is freed. + session = None diff --git a/tools/perf/python/compaction-times.py b/tools/perf/python/compaction-times.py new file mode 100755 index 000000000000..48d6af005c46 --- /dev/null +++ b/tools/perf/python/compaction-times.py @@ -0,0 +1,362 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +""" +Report time spent in memory compaction. + +Memory compaction is a feature in the Linux kernel that defragments memory +by moving used pages to create larger contiguous blocks of free memory. This +is particularly useful for allocating huge pages. + +This script processes trace events related to memory compaction and reports: +- Total time spent in compaction (stall time). +- Statistics for page migration (moved vs. failed). +- Statistics for the free scanner (scanned vs. isolated pages). +- Statistics for the migration scanner (scanned vs. isolated pages). + +Definitions: +- **Compaction**: Defragmenting memory by moving allocated pages. +- **Migration**: Moving pages from their current location to free pages found by the free scanner. +- **Free Scanner**: Scans memory (typically from the end of a zone) to find free pages. +- **Migration Scanner**: Scans memory (typically from the beginning of a zone) + to find pages to move. +- **Isolated Pages**: Pages that have been temporarily removed from the buddy + system for migration or as migration targets. + +Ported from tools/perf/scripts/python/compaction-times.py to the modern perf Python module. +""" +from __future__ import annotations + +import argparse +import enum +import re +import signal +import sys +from typing import Callable, Dict, List, Optional, Any +import perf + +if hasattr(signal, "SIGPIPE"): + signal.signal(signal.SIGPIPE, signal.SIG_DFL) + +class Popt(enum.IntEnum): + """Process display options.""" + DISP_DFL = 0 + DISP_PROC = 1 + DISP_PROC_VERBOSE = 2 + +class Topt(enum.IntFlag): + """Trace display options.""" + DISP_TIME = 0 + DISP_MIG = 1 + DISP_ISOLFREE = 2 + DISP_ISOLMIG = 4 + DISP_ALL = DISP_MIG | DISP_ISOLFREE | DISP_ISOLMIG + +# Globals to satisfy pylint when accessed in functions before assignment in main. +OPT_NS = True +opt_disp = Topt.DISP_ALL +opt_proc = Popt.DISP_DFL +session = None + +def get_comm_filter(regex: re.Pattern) -> Callable[[int, str], bool]: + """Returns a filter function based on command regex.""" + def filter_func(_pid: int, comm: str) -> bool: + regex_match = regex.search(comm) + return regex_match is None or regex_match.group() == "" + return filter_func + +def get_pid_filter(low_str: str, high_str: str) -> Callable[[int, str], bool]: + """Returns a filter function based on PID range.""" + low = 0 if low_str == "" else int(low_str) + high = None if high_str == "" else int(high_str) + + def filter_func(pid: int, _comm: str) -> bool: + return not (pid >= low and (high is None or pid <= high)) + return filter_func + +def ns_to_time(ns: int) -> str: + """Format nanoseconds to string based on options.""" + return f"{ns}ns" if OPT_NS else f"{round(ns, -3) // 1000}us" + +class Pair: + """Represents a pair of related counters (e.g., scanned vs isolated, moved vs failed).""" + def __init__(self, aval: int, bval: int, + alabel: Optional[str] = None, blabel: Optional[str] = None): + self.alabel = alabel + self.blabel = blabel + self.aval = aval + self.bval = bval + + def __add__(self, rhs: 'Pair') -> 'Pair': + return Pair(self.aval + rhs.aval, self.bval + rhs.bval, self.alabel, self.blabel) + + def __iadd__(self, rhs: 'Pair') -> 'Pair': + self.aval += rhs.aval + self.bval += rhs.bval + return self + + def __str__(self) -> str: + return f"{self.alabel}={self.aval} {self.blabel}={self.bval}" + +class Cnode: + """Holds statistics for a single compaction event or an aggregated set of events.""" + def __init__(self, ns: int): + self.ns = ns + self.migrated = Pair(0, 0, "moved", "failed") + self.fscan = Pair(0, 0, "scanned", "isolated") + self.mscan = Pair(0, 0, "scanned", "isolated") + + def __add__(self, rhs: 'Cnode') -> 'Cnode': + res = Cnode(self.ns + rhs.ns) + res.migrated = self.migrated + rhs.migrated + res.fscan = self.fscan + rhs.fscan + res.mscan = self.mscan + rhs.mscan + return res + + def __iadd__(self, rhs: 'Cnode') -> 'Cnode': + self.ns += rhs.ns + self.migrated += rhs.migrated + self.fscan += rhs.fscan + self.mscan += rhs.mscan + return self + + def __str__(self) -> str: + prev = False + s = f"{ns_to_time(self.ns)} " + if opt_disp & Topt.DISP_MIG: + s += f"migration: {self.migrated}" + prev = True + if opt_disp & Topt.DISP_ISOLFREE: + s += f"{' ' if prev else ''}free_scanner: {self.fscan}" + prev = True + if opt_disp & Topt.DISP_ISOLMIG: + s += f"{' ' if prev else ''}migration_scanner: {self.mscan}" + return s + + def complete(self, secs: int, nsecs: int) -> None: + """Complete the node with duration.""" + self.ns = (secs * 1000000000 + nsecs) - self.ns + + def increment(self, migrated: Optional[Pair], fscan: Optional[Pair], + mscan: Optional[Pair]) -> None: + """Increment statistics.""" + if migrated is not None: + self.migrated += migrated + if fscan is not None: + self.fscan += fscan + if mscan is not None: + self.mscan += mscan + +class Chead: + """Aggregates compaction statistics per process (PID) and maintains total statistics.""" + heads: Dict[int, 'Chead'] = {} + val = Cnode(0) + fobj: Optional[Any] = None + + @classmethod + def add_filter(cls, fobj: Any) -> None: + """Add a filter object.""" + cls.fobj = fobj + + @classmethod + def create_pending(cls, pid: int, comm: str, start_secs: int, start_nsecs: int) -> None: + """Create a pending node for a process.""" + filtered = False + try: + head = cls.heads[pid] + filtered = head.is_filtered() + except KeyError: + if cls.fobj is not None: + filtered = cls.fobj(pid, comm) + head = cls.heads[pid] = Chead(comm, pid, filtered) + + if not filtered: + head.mark_pending(start_secs, start_nsecs) + + @classmethod + def increment_pending(cls, pid: int, migrated: Optional[Pair], + fscan: Optional[Pair], mscan: Optional[Pair]) -> None: + """Increment pending stats for a process.""" + if pid not in cls.heads: + return + head = cls.heads[pid] + if not head.is_filtered(): + if head.is_pending(): + head.do_increment(migrated, fscan, mscan) + else: + sys.stderr.write(f"missing start compaction event for pid {pid}\n") + + @classmethod + def complete_pending(cls, pid: int, secs: int, nsecs: int) -> None: + """Complete pending stats for a process.""" + if pid not in cls.heads: + return + head = cls.heads[pid] + if not head.is_filtered(): + if head.is_pending(): + head.make_complete(secs, nsecs) + else: + sys.stderr.write(f"missing start compaction event for pid {pid}\n") + + @classmethod + def gen(cls): + """Generate heads for display.""" + if opt_proc != Popt.DISP_DFL: + yield from cls.heads.values() + + @classmethod + def get_total(cls) -> Cnode: + """Get total statistics.""" + return cls.val + + def __init__(self, comm: str, pid: int, filtered: bool): + self.comm = comm + self.pid = pid + self.val = Cnode(0) + self.pending: Optional[Cnode] = None + self.filtered = filtered + self.list: List[Cnode] = [] + + def mark_pending(self, secs: int, nsecs: int) -> None: + """Mark node as pending.""" + self.pending = Cnode(secs * 1000000000 + nsecs) + + def do_increment(self, migrated: Optional[Pair], fscan: Optional[Pair], + mscan: Optional[Pair]) -> None: + """Increment pending stats.""" + if self.pending is not None: + self.pending.increment(migrated, fscan, mscan) + + def make_complete(self, secs: int, nsecs: int) -> None: + """Make pending stats complete.""" + if self.pending is not None: + self.pending.complete(secs, nsecs) + Chead.val += self.pending + + if opt_proc != Popt.DISP_DFL: + self.val += self.pending + + if opt_proc == Popt.DISP_PROC_VERBOSE: + self.list.append(self.pending) + self.pending = None + + def enumerate(self) -> None: + """Enumerate verbose stats.""" + if opt_proc == Popt.DISP_PROC_VERBOSE and not self.is_filtered(): + for i, pelem in enumerate(self.list): + sys.stdout.write(f"{self.pid}[{self.comm}].{i+1}: {pelem}\n") + + def is_pending(self) -> bool: + """Check if node is pending.""" + return self.pending is not None + + def is_filtered(self) -> bool: + """Check if node is filtered.""" + return self.filtered + + def display(self) -> None: + """Display stats.""" + if not self.is_filtered(): + sys.stdout.write(f"{self.pid}[{self.comm}]: {self.val}\n") + +def trace_end() -> None: + """Called at the end of trace processing.""" + sys.stdout.write(f"total: {Chead.get_total()}\n") + for i in Chead.gen(): + i.display() + i.enumerate() + +def process_event(sample: perf.sample_event) -> None: + """Callback for processing events.""" + event_name = str(sample.evsel) + pid = sample.sample_tid + comm = "[unknown]" + try: + if session: + thread = session.find_thread(sample.sample_pid, pid) + if thread: + comm = thread.comm() or "[unknown]" + except (TypeError, AttributeError): + pass + secs = sample.sample_time // 1000000000 + nsecs = sample.sample_time % 1000000000 + + if "evsel(compaction:mm_compaction_begin)" in event_name: + Chead.create_pending(pid, comm, secs, nsecs) + elif "evsel(compaction:mm_compaction_end)" in event_name: + Chead.complete_pending(pid, secs, nsecs) + elif "evsel(compaction:mm_compaction_migratepages)" in event_name: + nr_migrated = getattr(sample, "nr_migrated", 0) + nr_failed = getattr(sample, "nr_failed", 0) + Chead.increment_pending(pid, Pair(nr_migrated, nr_failed), None, None) + elif "evsel(compaction:mm_compaction_isolate_freepages)" in event_name: + nr_scanned = getattr(sample, "nr_scanned", 0) + nr_taken = getattr(sample, "nr_taken", 0) + Chead.increment_pending(pid, None, Pair(nr_scanned, nr_taken), None) + elif "evsel(compaction:mm_compaction_isolate_migratepages)" in event_name: + nr_scanned = getattr(sample, "nr_scanned", 0) + nr_taken = getattr(sample, "nr_taken", 0) + Chead.increment_pending(pid, None, None, Pair(nr_scanned, nr_taken)) + +if __name__ == "__main__": + ap = argparse.ArgumentParser(description="Report time spent in compaction") + ap.add_argument("-p", action="store_true", help="display by process") + ap.add_argument("-pv", action="store_true", help="display by process (verbose)") + ap.add_argument("-u", action="store_true", help="display results in microseconds") + ap.add_argument("-t", action="store_true", help="display stall times only") + ap.add_argument("-m", action="store_true", help="display stats for migration") + ap.add_argument("-fs", action="store_true", help="display stats for free scanner") + ap.add_argument("-ms", action="store_true", help="display stats for migration scanner") + ap.add_argument("filter", nargs="?", help="pid|pid-range|comm-regex") + ap.add_argument("-i", "--input", default="perf.data", help="Input file name") + args = ap.parse_args() + + opt_proc = Popt.DISP_DFL + if args.pv: + opt_proc = Popt.DISP_PROC_VERBOSE + elif args.p: + opt_proc = Popt.DISP_PROC + + OPT_NS = not args.u + + opt_disp = Topt.DISP_ALL + if args.t or args.m or args.fs or args.ms: + opt_disp = Topt(0) + if args.t: + opt_disp |= Topt.DISP_TIME + if args.m: + opt_disp |= Topt.DISP_MIG + if args.fs: + opt_disp |= Topt.DISP_ISOLFREE + if args.ms: + opt_disp |= Topt.DISP_ISOLMIG + + if args.filter: + PID_PATTERN = r"^(\d*)-(\d*)$|^(\d*)$" + pid_re = re.compile(PID_PATTERN) + match = pid_re.search(args.filter) + filter_obj: Any = None + if match is not None and match.group() != "": + if match.group(3) is not None: + filter_obj = get_pid_filter(match.group(3), match.group(3)) + else: + filter_obj = get_pid_filter(match.group(1), match.group(2)) + else: + try: + comm_re = re.compile(args.filter) + except re.error: + sys.stderr.write(f"invalid regex '{args.filter}'\n") + sys.exit(1) + filter_obj = get_comm_filter(comm_re) + Chead.add_filter(filter_obj) + + try: + session = perf.session(perf.data(args.input), sample=process_event) + try: + session.process_events() + except KeyboardInterrupt: + pass + trace_end() + finally: + # Break the reference cycle between the global session and the + # process_event callback so the C perf.session object is freed. + session = None diff --git a/tools/perf/python/counting.py b/tools/perf/python/counting.py index 02121d2bb11d..9adbbeccdacd 100755 --- a/tools/perf/python/counting.py +++ b/tools/perf/python/counting.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 # SPDX-License-Identifier: GPL-2.0 +"""Example for counting perf events.""" # -*- python -*- # -*- coding: utf-8 -*- diff --git a/tools/perf/python/event_analyzing_sample.py b/tools/perf/python/event_analyzing_sample.py new file mode 100755 index 000000000000..b4efee88d94b --- /dev/null +++ b/tools/perf/python/event_analyzing_sample.py @@ -0,0 +1,317 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +""" +General event handler in Python, using SQLite to analyze events. + +The 2 database related functions in this script just show how to gather +the basic information, and users can modify and write their own functions +according to their specific requirement. + +The first function "show_general_events" just does a basic grouping for all +generic events with the help of sqlite, and the 2nd one "show_pebs_ll" is +for a x86 HW PMU event: PEBS with load latency data. + +Ported from tools/perf/scripts/python/event_analyzing_sample.py +""" +from __future__ import annotations + +import argparse +import math +import sqlite3 +import struct +from typing import Any +import perf + +# Event types, user could add more here +EVTYPE_GENERIC = 0 +EVTYPE_PEBS = 1 # Basic PEBS event +EVTYPE_PEBS_LL = 2 # PEBS event with load latency info +EVTYPE_IBS = 3 + +# +# Currently we don't have good way to tell the event type, but by +# the size of raw buffer, raw PEBS event with load latency data's +# size is 176 bytes, while the pure PEBS event's size is 144 bytes. +# +def create_event(name, comm, dso, symbol, raw_buf): + """Create an event object based on raw buffer size.""" + if len(raw_buf) == 144: + event = PebsEvent(name, comm, dso, symbol, raw_buf) + elif len(raw_buf) == 176: + event = PebsNHM(name, comm, dso, symbol, raw_buf) + else: + event = PerfEvent(name, comm, dso, symbol, raw_buf) + + return event + +class PerfEvent: + """Base class for all perf event samples.""" + event_num = 0 + def __init__(self, name, comm, dso, symbol, raw_buf, ev_type=EVTYPE_GENERIC): + self.name = name + self.comm = comm + self.dso = dso + self.symbol = symbol + self.raw_buf = raw_buf + self.ev_type = ev_type + PerfEvent.event_num += 1 + + def show(self): + """Display PMU event info.""" + print(f"PMU event: name={self.name:12s}, symbol={self.symbol:24s}, " + f"comm={self.comm:8s}, dso={self.dso:12s}") + +# +# Basic Intel PEBS (Precise Event-based Sampling) event, whose raw buffer +# contains the context info when that event happened: the EFLAGS and +# linear IP info, as well as all the registers. +# +class PebsEvent(PerfEvent): + """Intel PEBS event.""" + pebs_num = 0 + def __init__(self, name, comm, dso, symbol, raw_buf, ev_type=EVTYPE_PEBS): + tmp_buf = raw_buf[0:80] + flags, ip, ax, bx, cx, dx, si, di, bp, sp = struct.unpack('<QQQQQQQQQQ', tmp_buf) + self.flags = flags + self.ip = ip + self.ax = ax + self.bx = bx + self.cx = cx + self.dx = dx + self.si = si + self.di = di + self.bp = bp + self.sp = sp + + super().__init__(name, comm, dso, symbol, raw_buf, ev_type) + PebsEvent.pebs_num += 1 + del tmp_buf + +# +# Intel Nehalem and Westmere support PEBS plus Load Latency info which lie +# in the four 64 bit words write after the PEBS data: +# Status: records the IA32_PERF_GLOBAL_STATUS register value +# DLA: Data Linear Address (EIP) +# DSE: Data Source Encoding, where the latency happens, hit or miss +# in L1/L2/L3 or IO operations +# LAT: the actual latency in cycles +# +class PebsNHM(PebsEvent): + """Intel Nehalem/Westmere PEBS event with load latency.""" + pebs_nhm_num = 0 + def __init__(self, name, comm, dso, symbol, raw_buf, ev_type=EVTYPE_PEBS_LL): + tmp_buf = raw_buf[144:176] + status, dla, dse, lat = struct.unpack('<QQQQ', tmp_buf) + self.status = status + self.dla = dla + self.dse = dse + self.lat = lat + + super().__init__(name, comm, dso, symbol, raw_buf, ev_type) + PebsNHM.pebs_nhm_num += 1 + del tmp_buf + +session: Any = None + +class _DB: + con: sqlite3.Connection | None = None + +def trace_begin(db_path: str | None = None) -> None: + """Initialize database tables.""" + print("In trace_begin:\n") + con = sqlite3.connect(db_path or ":memory:") + try: + # Drop any pre-existing tables so repeated runs do not accumulate duplicate events. + con.execute("drop table if exists gen_events;") + con.execute("drop table if exists pebs_ll;") + + # Will create several tables at the start, pebs_ll is for PEBS data with + # load latency info, while gen_events is for general event. + con.execute(""" + create table if not exists gen_events ( + name text, + symbol text, + comm text, + dso text + );""") + con.execute(""" + create table if not exists pebs_ll ( + name text, + symbol text, + comm text, + dso text, + flags integer, + ip integer, + status integer, + dse integer, + dla integer, + lat integer + );""") + except BaseException: + con.close() + raise + _DB.con = con + +def insert_db(event: Any) -> None: + """Insert event into database.""" + con = _DB.con + assert con is not None + if event.ev_type == EVTYPE_GENERIC: + con.execute("insert into gen_events values(?, ?, ?, ?)", + (event.name, event.symbol, event.comm, event.dso)) + elif event.ev_type == EVTYPE_PEBS_LL: + # Python's sqlite3 module only accepts signed 64-bit integers (-2^63 to 2^63 - 1) + # and raises OverflowError for unsigned 64-bit kernel pointers (> 0x7fffffffffffffff). + # Convert unsigned 64-bit addresses to signed 64-bit two's-complement integers. + ip = event.ip - 0x10000000000000000 if event.ip > 0x7fffffffffffffff else event.ip + status = (event.status - 0x10000000000000000 + if event.status > 0x7fffffffffffffff else event.status) + dla = event.dla - 0x10000000000000000 if event.dla > 0x7fffffffffffffff else event.dla + con.execute("insert into pebs_ll values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + (event.name, event.symbol, event.comm, event.dso, event.flags, + ip, status, event.dse, dla, event.lat)) + +def process_event(sample: perf.sample_event) -> None: + """Callback for processing events.""" + # Create and insert event object to a database so that user could + # do more analysis with simple database commands. + + # Resolve comm, symbol, dso + comm = "Unknown_comm" + try: + if session is not None: + proc = session.find_thread(sample.sample_pid, sample.sample_tid) + if proc: + comm = proc.comm() or "Unknown_comm" + except TypeError: + pass + + # Symbol and dso info are not always resolved + dso = sample.dso if hasattr(sample, 'dso') and sample.dso else "Unknown_dso" + symbol = sample.symbol if hasattr(sample, 'symbol') and sample.symbol else "Unknown_symbol" + name = str(sample.evsel) + if name.startswith("evsel("): + name = name[6:-1] + + # Create the event object and insert it to the right table in database + try: + event = create_event(name, comm, dso, symbol, sample.raw_buf) + insert_db(event) + except (sqlite3.Error, ValueError, TypeError, OverflowError) as e: + print(f"Error creating/inserting event: {e}") + +def num2sym(num: int) -> str: + """Convert number to a histogram symbol (log2).""" + # As the event number may be very big, so we can't use linear way + # to show the histogram in real number, but use a log2 algorithm. + if num <= 0: + return "" + snum = '#' * (int(math.log(num, 2)) + 1) + return snum + +def show_general_events() -> None: + """Display statistics for general events.""" + con = _DB.con + assert con is not None + count = con.execute("select count(*) from gen_events") + for t in count: + print(f"There is {t[0]} records in gen_events table") + if t[0] == 0: + return + + print("Statistics about the general events grouped by thread/symbol/dso: \n") + + # Group by thread + commq = con.execute(""" + select comm, count(comm) from gen_events + group by comm order by -count(comm) + """) + print(f"\n{ 'comm':>16} {'number':>8} {'histogram':>16}\n{'='*42}") + for row in commq: + print(f"{row[0]:>16} {row[1]:>8} {num2sym(row[1])}") + + # Group by symbol + print(f"\n{'symbol':>32} {'number':>8} {'histogram':>16}\n{'='*58}") + symbolq = con.execute(""" + select symbol, count(symbol) from gen_events + group by symbol order by -count(symbol) + """) + for row in symbolq: + print(f"{row[0]:>32} {row[1]:>8} {num2sym(row[1])}") + + # Group by dso + print(f"\n{'dso':>40} {'number':>8} {'histogram':>16}\n{'='*74}") + dsoq = con.execute("select dso, count(dso) from gen_events group by dso order by -count(dso)") + for row in dsoq: + print(f"{row[0]:>40} {row[1]:>8} {num2sym(row[1])}") + +def show_pebs_ll() -> None: + """Display statistics for PEBS load latency events.""" + con = _DB.con + assert con is not None + # This function just shows the basic info, and we could do more with the + # data in the tables, like checking the function parameters when some + # big latency events happen. + count = con.execute("select count(*) from pebs_ll") + for t in count: + print(f"There is {t[0]} records in pebs_ll table") + if t[0] == 0: + return + + print("Statistics about the PEBS Load Latency events grouped by thread/symbol/dse/latency: \n") + + # Group by thread + commq = con.execute("select comm, count(comm) from pebs_ll group by comm order by -count(comm)") + print(f"\n{'comm':>16} {'number':>8} {'histogram':>16}\n{'='*42}") + for row in commq: + print(f"{row[0]:>16} {row[1]:>8} {num2sym(row[1])}") + + # Group by symbol + print(f"\n{'symbol':>32} {'number':>8} {'histogram':>16}\n{'='*58}") + symbolq = con.execute(""" + select symbol, count(symbol) from pebs_ll + group by symbol order by -count(symbol) + """) + for row in symbolq: + print(f"{row[0]:>32} {row[1]:>8} {num2sym(row[1])}") + + # Group by dse + dseq = con.execute("select dse, count(dse) from pebs_ll group by dse order by -count(dse)") + print(f"\n{'dse':>32} {'number':>8} {'histogram':>16}\n{'='*58}") + for row in dseq: + print(f"{row[0]:>32} {row[1]:>8} {num2sym(row[1])}") + + # Group by latency + latq = con.execute("select lat, count(lat) from pebs_ll group by lat order by lat") + print(f"\n{'latency':>32} {'number':>8} {'histogram':>16}\n{'='*58}") + for row in latq: + print(f"{str(row[0]):>32} {row[1]:>8} {num2sym(row[1])}") + +def trace_end() -> None: + """Called at the end of trace processing.""" + print("In trace_end:\n") + if _DB.con: + try: + _DB.con.commit() + show_general_events() + show_pebs_ll() + finally: + _DB.con.close() + _DB.con = None + +if __name__ == "__main__": + ap = argparse.ArgumentParser(description="Analyze events with SQLite") + ap.add_argument("-i", "--input", default="perf.data", help="Input file name") + ap.add_argument("-d", "--db", "--database", dest="database", default=None, + help="Database file name (defaults to an in-memory database)") + args = ap.parse_args() + + try: + trace_begin(args.database) + session = perf.session(perf.data(args.input), sample=process_event) + session.process_events() + finally: + # Break the reference cycle between session and process_event (whose module + # globals reference session) so the C session object is deterministically freed. + session = None + trace_end() diff --git a/tools/perf/python/export-to-postgresql.py b/tools/perf/python/export-to-postgresql.py new file mode 100755 index 000000000000..9cac64dee361 --- /dev/null +++ b/tools/perf/python/export-to-postgresql.py @@ -0,0 +1,1328 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +r""" +Export perf data to a postgresql database. + +This script has been ported to use the modern perf Python module and +libpq via ctypes. It no longer requires PySide2 or QtSql for exporting. + +The script assumes postgresql is running on the local machine and that the +user has postgresql permissions to create databases. + +An example of using this script with Intel PT: + + $ perf record -e intel_pt//u ls + $ python export-to-postgresql.py -i perf.data -o pt_example + +To browse the database, psql can be used e.g. + + $ psql pt_example + pt_example=# select * from samples_view where id < 100; + pt_example=# \d+ + pt_example=# \d+ samples_view + pt_example=# \q + +An example of using the database is provided by the script +exported-sql-viewer.py. Refer to that script for details. + +Tables: + + The tables largely correspond to perf tools' data structures. They are + largely self-explanatory. + + samples + 'samples' is the main table. It represents what instruction was + executing at a point in time when something (a selected event) + happened. The memory address is the instruction pointer or 'ip'. + + branch_types + 'branch_types' provides descriptions for each type of branch. + + comm_threads + 'comm_threads' shows how 'comms' relates to 'threads'. + + comms + 'comms' contains a record for each 'comm' - the name given to the + executable that is running. + + dsos + 'dsos' contains a record for each executable file or library. + + machines + 'machines' can be used to distinguish virtual machines if + virtualization is supported. + + selected_events + 'selected_events' contains a record for each kind of event that + has been sampled. + + symbols + 'symbols' contains a record for each symbol. Only symbols that + have samples are present. + + threads + 'threads' contains a record for each thread. + +Views: + + Most of the tables have views for more friendly display. The views are: + + comm_threads_view + dsos_view + machines_view + samples_view + symbols_view + threads_view + +Ported from tools/perf/scripts/python/export-to-postgresql.py +""" + +from __future__ import annotations +import typing +import argparse +from ctypes import CDLL, c_char_p, c_int, c_void_p, c_ubyte +import ctypes.util +import os +import shutil +import struct +import tempfile +import sys +from typing import Any, Dict, Optional +import perf + +# Need to access PostgreSQL C library directly to use COPY FROM STDIN +libpq_name = ctypes.util.find_library("pq") +if not libpq_name: + libpq_name = "libpq.so.5" + +try: + libpq = CDLL(libpq_name) +except OSError as e: + print(f"Error loading {libpq_name}: {e}") + print("Please ensure PostgreSQL client library is installed.") + sys.exit(1) + +PQconnectdb = libpq.PQconnectdb +PQconnectdb.restype = c_void_p +PQconnectdb.argtypes = [c_char_p] +PQfinish = libpq.PQfinish +PQfinish.argtypes = [c_void_p] +PQstatus = libpq.PQstatus +PQstatus.restype = c_int +PQstatus.argtypes = [c_void_p] +libpq.PQerrorMessage.restype = c_char_p +libpq.PQerrorMessage.argtypes = [c_void_p] +PQexec = libpq.PQexec +PQexec.restype = c_void_p +PQexec.argtypes = [c_void_p, c_char_p] +PQresultStatus = libpq.PQresultStatus +PQresultStatus.restype = c_int +PQresultStatus.argtypes = [c_void_p] +PQputCopyData = libpq.PQputCopyData +PQputCopyData.restype = c_int +PQputCopyData.argtypes = [c_void_p, c_void_p, c_int] +PQputCopyEnd = libpq.PQputCopyEnd +PQputCopyEnd.restype = c_int +PQputCopyEnd.argtypes = [c_void_p, c_void_p] +PQgetResult = libpq.PQgetResult +PQgetResult.restype = c_void_p +PQgetResult.argtypes = [c_void_p] +PQclear = libpq.PQclear +PQclear.argtypes = [c_void_p] + + +def toserverstr(s: str) -> bytes: + """Convert string to server encoding (UTF-8).""" + return bytes(s, "UTF_8") + + +def toclientstr(s: str) -> bytes: + """Convert string to client encoding (UTF-8).""" + return bytes(s, "UTF_8") + +PERF_IP_FLAG_BRANCH = 1 << 0 +PERF_IP_FLAG_CALL = 1 << 1 +PERF_IP_FLAG_RETURN = 1 << 2 +PERF_IP_FLAG_CONDITIONAL = 1 << 3 +PERF_IP_FLAG_SYSCALLRET = 1 << 4 +PERF_IP_FLAG_ASYNC = 1 << 5 +PERF_IP_FLAG_INTERRUPT = 1 << 6 +PERF_IP_FLAG_TX_ABORT = 1 << 7 +PERF_IP_FLAG_TRACE_BEGIN = 1 << 8 +PERF_IP_FLAG_TRACE_END = 1 << 9 +PERF_IP_FLAG_IN_TX = 1 << 10 +PERF_IP_FLAG_VMENTRY = 1 << 11 +PERF_IP_FLAG_VMEXIT = 1 << 12 + +BRANCH_TYPES = [ + (0, "no branch"), + (PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL, "call"), + (PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_RETURN, "return"), + (PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CONDITIONAL, "conditional jump"), + (PERF_IP_FLAG_BRANCH, "unconditional jump"), + (PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL | PERF_IP_FLAG_INTERRUPT, "software interrupt"), + (PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_RETURN | PERF_IP_FLAG_INTERRUPT, "return from interrupt"), + (PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL | PERF_IP_FLAG_SYSCALLRET, "system call"), + (PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_RETURN | PERF_IP_FLAG_SYSCALLRET, + "return from system call"), + (PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_ASYNC, "asynchronous branch"), + (PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL | PERF_IP_FLAG_ASYNC | PERF_IP_FLAG_INTERRUPT, + "hardware interrupt"), + (PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_TX_ABORT, "transaction abort"), + (PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_TRACE_BEGIN, "trace begin"), + (PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_TRACE_END, "trace end"), + (PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL | PERF_IP_FLAG_VMENTRY, "vm entry"), + (PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL | PERF_IP_FLAG_VMEXIT, "vm exit"), +] +for _b_type, _b_name in list(BRANCH_TYPES): + if (_b_type == PERF_IP_FLAG_BRANCH or + (_b_type & (PERF_IP_FLAG_TRACE_BEGIN | PERF_IP_FLAG_TRACE_END))): + continue + BRANCH_TYPES.append((_b_type | PERF_IP_FLAG_TRACE_BEGIN, f"trace begin / {_b_name}")) + BRANCH_TYPES.append((_b_type | PERF_IP_FLAG_TRACE_END, f"{_b_name} / trace end")) + +class PostgresExporter: + """Handles PostgreSQL connection and exporting of perf events.""" + + def __init__(self, dbname: str): + self.dbname = dbname + self.conn = None + self.session: Optional[perf.session] = None + self.output_dir_name = tempfile.mkdtemp(prefix=dbname + "-perf-data-") + self.created_output_dir = True + + self.file_header = struct.pack("!11sii", b"PGCOPY\n\377\r\n\0", 0, 0) + self.file_trailer = b"\377\377" + + # Caches and counters grouped to reduce instance attributes + self.caches: Dict[str, dict] = { + 'machines': {}, + 'threads': {}, + 'comms': {}, + 'dsos': {}, + 'symbols': {}, + 'events': {}, + 'branch_types': {}, + 'call_paths': {} + } + self.symbol_rows: Dict[int, tuple[int, int, int, int, str]] = {} + + self.next_id = { + 'machine': 1, + 'thread': 1, + 'comm': 1, + 'dso': 1, + 'symbol': 1, + 'event': 1, + 'branch_type': 1, + 'comm_thread': 1, + 'call_path': 1, + 'sample': 1, + 'call': 1, + 'context_switch': 1 + } + + self.files: Dict[str, Any] = {} + self.unhandled_count = 0 + + def connect(self, db_to_use: str) -> None: + """Connect to database. + + Reject URI schemes ('://') and key=value pairs ('=') to prevent + connection parameter injection, and escape backslashes and single + quotes per libpq single-quoted conninfo string rules. + """ + if "://" in db_to_use or "=" in db_to_use: + raise ValueError(f"Invalid database name: {db_to_use}") + safe_db = db_to_use.replace('\\', '\\\\').replace("'", "\\'") + conn_str = toclientstr(f"dbname='{safe_db}'") + self.conn = PQconnectdb(conn_str) + if PQstatus(self.conn) != 0: + PQfinish(self.conn) + self.conn = None + raise RuntimeError(f"PQconnectdb failed for {db_to_use}") + + def disconnect(self) -> None: + """Disconnect from database.""" + if self.conn: + PQfinish(self.conn) + self.conn = None + + def do_query(self, sql: str) -> None: + """Execute a query and check status.""" + res = PQexec(self.conn, toserverstr(sql)) + status = PQresultStatus(res) + PQclear(res) + if status not in (1, 2): # PGRES_COMMAND_OK, PGRES_TUPLES_OK + error_msg = libpq.PQerrorMessage(self.conn).decode('utf-8') + raise RuntimeError(f"Query failed: {sql}. Error: {error_msg}") + + def open_output_file(self, file_name: str): + """Open intermediate binary file.""" + path_name = self.output_dir_name + "/" + file_name + f = open(path_name, "wb+") + f.write(self.file_header) + return f + + def close_output_file(self, f): + """Close intermediate binary file.""" + f.write(self.file_trailer) + f.close() + + def copy_output_file(self, path_name: str, table_name: str): + """Copy intermediate file to database.""" + sql = f"COPY {table_name} FROM STDIN (FORMAT 'binary')" + res = PQexec(self.conn, toserverstr(sql)) + if PQresultStatus(res) != 4: # PGRES_COPY_IN + PQclear(res) + err = libpq.PQerrorMessage(self.conn).decode() + raise RuntimeError(f"COPY FROM STDIN PQexec failed for {table_name}: {err}") + PQclear(res) + + with open(path_name, "rb") as f: + data = f.read(65536) + while len(data) > 0: + c_data = (c_ubyte * len(data)).from_buffer_copy(data) + ret = PQputCopyData(self.conn, c_data, len(data)) + if ret != 1: + raise RuntimeError(f"PQputCopyData failed for {table_name}") + data = f.read(65536) + + ret = PQputCopyEnd(self.conn, None) + if ret != 1: + err = libpq.PQerrorMessage(self.conn).decode() + raise RuntimeError(f"PQputCopyEnd failed for {table_name}: {err}") + + res = PQgetResult(self.conn) + while res: + status = PQresultStatus(res) + if status != 1: # PGRES_COMMAND_OK + error_msg = libpq.PQerrorMessage(self.conn).decode('utf-8') + PQclear(res) + raise RuntimeError( + f"COPY completion failed for {table_name}. Status {status}: {error_msg}" + ) + PQclear(res) + res = PQgetResult(self.conn) + + def setup_db(self) -> None: + """Create database and tables. MUST be called after init.""" + if "://" in self.dbname or "=" in self.dbname: + raise ValueError(f"Invalid database name: {self.dbname}") + self.created_output_dir = True + + self.connect('postgres') + try: + # SQL identifiers are double-quoted; escape embedded double quotes by doubling them. + db_name = self.dbname.replace('"', '""') + self.do_query(f'CREATE DATABASE "{db_name}"') + except Exception as db_error: + shutil.rmtree(self.output_dir_name, ignore_errors=True) + raise db_error + self.disconnect() + + self.connect(self.dbname) + self.do_query("SET client_min_messages TO WARNING") + + self.do_query(""" + CREATE TABLE selected_events ( + id bigint NOT NULL, + name varchar(80)) + """) + self.do_query(""" + CREATE TABLE machines ( + id bigint NOT NULL, + pid integer, + root_dir varchar(4096)) + """) + self.do_query(""" + CREATE TABLE threads ( + id bigint NOT NULL, + machine_id bigint, + process_id bigint, + pid integer, + tid integer) + """) + self.do_query(""" + CREATE TABLE comms ( + id bigint NOT NULL, + comm varchar(16), + c_thread_id bigint, + c_time bigint, + exec_flag boolean) + """) + self.do_query(""" + CREATE TABLE comm_threads ( + id bigint NOT NULL, + comm_id bigint, + thread_id bigint) + """) + self.do_query(""" + CREATE TABLE dsos ( + id bigint NOT NULL, + machine_id bigint, + short_name varchar(256), + long_name varchar(4096), + build_id varchar(64)) + """) + self.do_query(""" + CREATE TABLE symbols ( + id bigint NOT NULL, + dso_id bigint, + sym_start bigint, + sym_end bigint, + binding integer, + name varchar(2048)) + """) + self.do_query(""" + CREATE TABLE branch_types ( + id integer NOT NULL, + name varchar(80)) + """) + self.do_query(""" + CREATE TABLE samples ( + id bigint NOT NULL, + evsel_id bigint, + machine_id bigint, + thread_id bigint, + comm_id bigint, + dso_id bigint, + symbol_id bigint, + sym_offset bigint, + ip bigint, + time bigint, + cpu integer, + to_dso_id bigint, + to_symbol_id bigint, + to_sym_offset bigint, + to_ip bigint, + period bigint, + weight bigint, + transaction_ bigint, + data_src bigint, + branch_type integer, + in_tx boolean, + call_path_id bigint, + insn_count bigint, + cyc_count bigint, + flags integer) + """) + self.do_query(''' + CREATE TABLE context_switches ( + id bigint, + machine_id bigint, + time bigint, + cpu integer, + thread_out_id bigint, + comm_out_id bigint, + thread_in_id bigint, + comm_in_id bigint, + flags integer + ) + ''') + self.do_query(''' + CREATE TABLE call_paths ( + id bigint NOT NULL, + parent_id bigint, + symbol_id bigint, + ip bigint) + ''') + self.do_query(''' + CREATE TABLE calls ( + id bigint NOT NULL, + thread_id bigint, + comm_id bigint, + call_path_id bigint, + call_time bigint, + return_time bigint, + branch_count bigint, + call_id bigint, + return_id bigint, + parent_call_path_id bigint, + flags integer, + parent_id bigint, + insn_count bigint, + cyc_count bigint + ) + ''') + self.do_query(''' + CREATE TABLE ptwrite ( + id bigint NOT NULL, + payload bigint, + exact_ip boolean + ) + ''') + self.do_query(''' + CREATE TABLE cbr ( + id bigint NOT NULL, + cbr integer, + mhz integer, + percent integer + ) + ''') + self.do_query(''' + CREATE TABLE mwait ( + id bigint NOT NULL, + hints integer, + extensions integer + ) + ''') + self.do_query(''' + CREATE TABLE pwre ( + id bigint NOT NULL, + cstate integer, + subcstate integer, + hw boolean + ) + ''') + self.do_query(''' + CREATE TABLE exstop ( + id bigint NOT NULL, + exact_ip boolean + ) + ''') + self.do_query(''' + CREATE TABLE pwrx ( + id bigint NOT NULL, + deepest_cstate integer, + last_cstate integer, + wake_reason integer + ) + ''') + + self.files['evsel'] = self.open_output_file("evsel_table.bin") + self.files['machine'] = self.open_output_file("machine_table.bin") + self.files['thread'] = self.open_output_file("thread_table.bin") + self.files['comm'] = self.open_output_file("comm_table.bin") + self.files['comm_thread'] = self.open_output_file("comm_thread_table.bin") + self.files['dso'] = self.open_output_file("dso_table.bin") + self.files['symbol'] = self.open_output_file("symbol_table.bin") + self.files['branch_type'] = self.open_output_file("branch_type_table.bin") + self.files['sample'] = self.open_output_file("sample_table.bin") + self.files['context_switches'] = self.open_output_file("context_switches_table.bin") + self.files['call'] = self.open_output_file("call_table.bin") + self.files['call_path'] = self.open_output_file("call_path_table.bin") + self.files['ptwrite'] = self.open_output_file("ptwrite_table.bin") + self.files['cbr'] = self.open_output_file("cbr_table.bin") + self.files['mwait'] = self.open_output_file("mwait_table.bin") + self.files['pwre'] = self.open_output_file("pwre_table.bin") + self.files['exstop'] = self.open_output_file("exstop_table.bin") + self.files['pwrx'] = self.open_output_file("pwrx_table.bin") + + self.write_evsel(0, "unknown") + self.write_machine(0, 0, "unknown") + self.write_thread(0, 0, 0, -1, -1) + self.write_comm(0, "unknown", 0, 0, 0) + self.write_dso(0, 0, "unknown", "unknown", "") + self.write_symbol(0, 0, 0, 0, 0, "unknown") + self.write_call_path(0, 0, 0, 0) + # Sentinel id = 0 rows in sample_table.bin and call_table.bin satisfy + # PostgreSQL FOREIGN KEY constraints (call_id/return_id -> samples.id + # and parent references) when a call or return event has no matching + # sample reference (0). + self.files['sample'].write(struct.pack( + "!hiqiqiqiqiqiqiqiQiQiQiIiqiqiQiQiQiQiQiQiiiBiqiQiQii", + 25, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 4, 0, + 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 4, 0, 1, 0, 8, 0, 8, 0, 8, 0, 4, 0 + )) + self.files['call'].write(struct.pack( + "!hiqiqiqiqiqiqiqiqiqiqiiiqiqiq", + 14, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 8, 0, 4, 0, 8, 0, 8, 0, 8, 0 + )) + for b_type, b_name in BRANCH_TYPES: + self.write_branch_type(b_type, b_name) + + def write_branch_type(self, branch_type: int, name: str) -> None: + name_bytes = toserverstr(name) + n = len(name_bytes) + fmt = "!hiii" + str(n) + "s" + value = struct.pack(fmt, 2, 4, branch_type, n, name_bytes) + self.files['branch_type'].write(value) + + def write_ptwrite(self, id_: int, raw_buf: bytes) -> None: + data = struct.unpack_from("<IQ", raw_buf) + flags, payload = data[0], data[1] + value = struct.pack("!hiqiQiB", 3, 8, id_, 8, payload, 1, flags & 1) + self.files['ptwrite'].write(value) + + def write_cbr(self, id_: int, raw_buf: bytes) -> None: + data = struct.unpack_from("<BBBBII", raw_buf) + cbr_val, max_nonturbo, freq = data[0], data[2], data[4] + mhz = (freq + 500) // 1000 + percent = ((cbr_val * 1000 // max_nonturbo) + 5) // 10 if max_nonturbo else 0 + value = struct.pack("!hiqiiiiii", 4, 8, id_, 4, cbr_val, 4, int(mhz), 4, int(percent)) + self.files['cbr'].write(value) + + def write_mwait(self, id_: int, raw_buf: bytes) -> None: + data = struct.unpack_from("<IQ", raw_buf) + payload = data[1] + hints = payload & 0xff + extensions = (payload >> 32) & 0x3 + value = struct.pack("!hiqiiii", 3, 8, id_, 4, hints, 4, extensions) + self.files['mwait'].write(value) + + def write_pwre(self, id_: int, raw_buf: bytes) -> None: + data = struct.unpack_from("<IQ", raw_buf) + payload = data[1] + hw = (payload >> 7) & 1 + cstate = (payload >> 12) & 0xf + subcstate = (payload >> 8) & 0xf + value = struct.pack("!hiqiiiiiB", 4, 8, id_, 4, cstate, 4, subcstate, 1, hw) + self.files['pwre'].write(value) + + def write_exstop(self, id_: int, raw_buf: bytes) -> None: + data = struct.unpack_from("<I", raw_buf) + flags = data[0] + value = struct.pack("!hiqiB", 2, 8, id_, 1, flags & 1) + self.files['exstop'].write(value) + + def write_pwrx(self, id_: int, raw_buf: bytes) -> None: + data = struct.unpack_from("<IQ", raw_buf) + payload = data[1] + deepest = payload & 0xf + last = (payload >> 4) & 0xf + wake = (payload >> 8) & 0xf + value = struct.pack("!hiqiiiiii", 4, 8, id_, 4, deepest, 4, last, 4, wake) + self.files['pwrx'].write(value) + + def write_evsel(self, evsel_id: int, name: str) -> None: + """Write event to binary file.""" + name_bytes = toserverstr(name) + n = len(name_bytes) + fmt = "!hiqi" + str(n) + "s" + value = struct.pack(fmt, 2, 8, evsel_id, n, name_bytes) + self.files['evsel'].write(value) + + def write_machine(self, machine_id: int, pid: int, root_dir: str) -> None: + """Write machine to binary file.""" + rd_bytes = toserverstr(root_dir) + n = len(rd_bytes) + fmt = "!hiqiii" + str(n) + "s" + value = struct.pack(fmt, 3, 8, machine_id, 4, pid, n, rd_bytes) + self.files['machine'].write(value) + + def write_thread(self, thread_id: int, machine_id: int, process_id: int, + pid: int, tid: int) -> None: + """Write thread to binary file.""" + value = struct.pack("!hiqiqiqiIiI", 5, 8, thread_id, 8, machine_id, + 8, process_id, 4, pid & 0xffffffff, 4, tid & 0xffffffff) + self.files['thread'].write(value) + + def write_comm(self, comm_id: int, comm_str: str, thread_id: int, + time: int, exec_flag: int) -> None: + """Write comm to binary file.""" + comm_bytes = toserverstr(comm_str) + n = len(comm_bytes) + fmt = "!hiqi" + str(n) + "s" + "iqiqiB" + value = struct.pack(fmt, 5, 8, comm_id, n, comm_bytes, 8, + thread_id, 8, time, 1, exec_flag) + self.files['comm'].write(value) + + def write_comm_thread(self, comm_thread_id: int, comm_id: int, + thread_id: int) -> None: + """Write comm_thread to binary file.""" + fmt = "!hiqiqiq" + value = struct.pack(fmt, 3, 8, comm_thread_id, 8, comm_id, 8, thread_id) + self.files['comm_thread'].write(value) + + def write_dso(self, dso_id: int, machine_id: int, short_name: str, + long_name: str, build_id: str) -> None: + """Write DSO to binary file.""" + sn_bytes = toserverstr(short_name) + ln_bytes = toserverstr(long_name) + bi_bytes = toserverstr(build_id) + n1, n2, n3 = len(sn_bytes), len(ln_bytes), len(bi_bytes) + fmt = "!hiqiqi" + str(n1) + "si" + str(n2) + "si" + str(n3) + "s" + value = struct.pack(fmt, 5, 8, dso_id, 8, machine_id, n1, + sn_bytes, n2, ln_bytes, n3, bi_bytes) + self.files['dso'].write(value) + + def write_symbol(self, symbol_id: int, dso_id: int, sym_start: int, + sym_end: int, binding: int, symbol_name: str) -> None: + """Write symbol to binary file.""" + name_bytes = toserverstr(symbol_name) + n = len(name_bytes) + fmt = "!hiqiqiQiQiii" + str(n) + "s" + value = struct.pack(fmt, 6, 8, symbol_id, 8, dso_id, 8, + sym_start, 8, sym_end, 4, binding, n, name_bytes) + self.files['symbol'].write(value) + + def write_call_path(self, cp_id: int, parent_id: int, symbol_id: int, + ip: int) -> None: + """Write call path to binary file.""" + fmt = "!hiqiqiqiQ" + value = struct.pack(fmt, 4, 8, cp_id, 8, parent_id, 8, symbol_id, 8, ip) + self.files['call_path'].write(value) + + def write_sample(self, sample_id: int, evsel_id: int, machine_id: int, thread_id: int, + comm_id: int, dso_id: int, symbol_id: int, + sample: perf.sample_event, call_path_id: int) -> None: + """Write sample to binary file.""" + addr_dso_id = self.get_dso_id( + sample.addr_dso or "Unknown_dso", sample.addr_dso or "Unknown_dso_long", "", + machine_id + ) + addr_symbol_id = self.get_symbol_id( + addr_dso_id, sample.addr_symbol or "Unknown_symbol", 0, 0 + ) + value = struct.pack( + "!hiqiqiqiqiqiqiqiQiQiQiIiqiqiQiQiQiQiQiQiiiBiqiQiQii", + 25, 8, sample_id, 8, evsel_id, 8, machine_id, 8, thread_id, 8, comm_id, + 8, dso_id, 8, symbol_id, 8, getattr(sample, 'sym_offset', 0) or 0, + 8, getattr(sample, 'sample_ip', 0) or 0, + 8, getattr(sample, 'sample_time', 0) or 0, + 4, getattr(sample, 'sample_cpu', 0) & 0xffffffff, + 8, addr_dso_id, + 8, addr_symbol_id, + 8, getattr(sample, 'addr_sym_offset', 0) or 0, + 8, getattr(sample, 'sample_addr', 0) or 0, + 8, getattr(sample, 'sample_period', 0) or 0, + 8, getattr(sample, 'sample_weight', 0) or 0, + 8, getattr(sample, 'transaction', 0) or 0, + 8, getattr(sample, 'sample_data_src', 0) or 0 or 0, + 4, getattr(sample, 'branch_type', 0) or 0, + 1, getattr(sample, 'in_tx', 0) or 0, + 8, call_path_id, + 8, getattr(sample, 'sample_insn_count', 0) or 0 or 0, + 8, getattr(sample, 'sample_cyc_count', 0) or 0 or 0, + 4, getattr(sample, 'flags', 0) or 0 + ) + self.files['sample'].write(value) + + def get_machine_id(self, machine_pid: Optional[int]) -> int: + """Get or create machine ID.""" + if machine_pid is None or machine_pid <= 0 or machine_pid > 0x7fffffff: + machine_pid = -1 + if machine_pid in self.caches['machines']: + return self.caches['machines'][machine_pid] + machine_id = self.next_id['machine'] + self.next_id['machine'] += 1 + self.write_machine(machine_id, machine_pid, "") + self.caches['machines'][machine_pid] = machine_id + return machine_id + + def get_event_id(self, name: str) -> int: + """Get or create event ID.""" + if name in self.caches['events']: + return self.caches['events'][name] + event_id = self.next_id['event'] + self.write_evsel(event_id, name) + self.caches['events'][name] = event_id + self.next_id['event'] += 1 + return event_id + + def get_thread_id(self, machine_id: int, pid: Optional[int], tid: Optional[int]) -> int: + """Get or create thread ID.""" + if pid is None: + pid = -1 + elif pid > 0x7fffffff: + pid -= 0x100000000 + if tid is None: + tid = -1 + elif tid > 0x7fffffff: + tid -= 0x100000000 + key = (machine_id, pid, tid) + if key in self.caches['threads']: + return self.caches['threads'][key] + process_id = self.get_thread_id(machine_id, pid, pid) if tid != pid else -1 + thread_id = self.next_id['thread'] + if process_id == -1: process_id = thread_id + self.write_thread(thread_id, machine_id, process_id, pid, tid) + self.caches['threads'][key] = thread_id + self.next_id['thread'] += 1 + return thread_id + + def get_comm_id(self, comm: str, thread_id: int) -> int: + """Get or create comm ID.""" + c_key = (comm, thread_id) + if c_key in self.caches['comms']: + comm_id = self.caches['comms'][c_key] + else: + comm_id = self.next_id['comm'] + self.write_comm(comm_id, comm, thread_id, 0, 0) + self.caches['comms'][c_key] = comm_id + self.next_id['comm'] += 1 + + key = (comm_id, thread_id) + if 'comm_threads' not in self.caches: + self.caches['comm_threads'] = {} + if key not in self.caches['comm_threads']: + comm_thread_id = self.next_id['comm_thread'] + self.write_comm_thread(comm_thread_id, comm_id, thread_id) + self.caches['comm_threads'][key] = True + self.next_id['comm_thread'] += 1 + + return comm_id + + def get_dso_id(self, short_name: str, long_name: str, + build_id: str, machine_id: int = 0) -> int: + """Get or create DSO ID.""" + key = (machine_id, short_name, long_name, build_id) + if key in self.caches['dsos']: + return self.caches['dsos'][key] + short_key = (machine_id, short_name) + if not build_id and short_key in self.caches['dsos']: + return self.caches['dsos'][short_key] + dso_id = self.next_id['dso'] + self.write_dso(dso_id, machine_id, short_name, long_name, build_id) + self.caches['dsos'][key] = dso_id + self.caches['dsos'][short_key] = dso_id + self.next_id['dso'] += 1 + return dso_id + + def get_symbol_id(self, dso_id: int, name: str, start: int, + end: int) -> int: + """Get or create symbol ID.""" + key = (dso_id, name) + if key in self.caches['symbols']: + symbol_id, cached_start, cached_end = self.caches['symbols'][key] + if cached_start == 0 and cached_end == 0 and (start != 0 or end != 0): + self.caches['symbols'][key] = (symbol_id, start, end) + self.symbol_rows[symbol_id] = (dso_id, start, end, 0, name) + return symbol_id + symbol_id = self.next_id['symbol'] + self.caches['symbols'][key] = (symbol_id, start, end) + self.symbol_rows[symbol_id] = (dso_id, start, end, 0, name) + self.next_id['symbol'] += 1 + return symbol_id + + def get_call_path_id(self, parent_id: int, symbol_id: int, + ip: int) -> int: + """Get or create call path ID.""" + key = (parent_id, symbol_id, ip) + if key in self.caches['call_paths']: + return self.caches['call_paths'][key] + call_path_id = self.next_id['call_path'] + self.write_call_path(call_path_id, parent_id, symbol_id, ip) + self.caches['call_paths'][key] = call_path_id + self.next_id['call_path'] += 1 + return call_path_id + + def process_context_switch(self, event: typing.Any) -> None: + """Callback for processing context switch events.""" + misc = getattr(event, 'misc', 0) + out = bool(misc & (1 << 13)) + out_preempt = bool(misc & (1 << 14)) + flags = (1 if out else 0) | ((1 if out_preempt else 0) << 1) + machine_id = self.get_machine_id(getattr(event, 'machine_pid', None)) + + sample_pid = getattr(event, 'sample_pid', None) + if sample_pid is None: + sample_pid = -1 + elif sample_pid > 0x7fffffff: + sample_pid -= 0x100000000 + sample_tid = getattr(event, 'sample_tid', None) + if sample_tid is None: + sample_tid = -1 + elif sample_tid > 0x7fffffff: + sample_tid -= 0x100000000 + next_prev_pid = getattr(event, 'next_prev_pid', None) + if next_prev_pid is None: + next_prev_pid = -1 + elif next_prev_pid > 0x7fffffff: + next_prev_pid -= 0x100000000 + next_prev_tid = getattr(event, 'next_prev_tid', None) + if next_prev_tid is None: + next_prev_tid = -1 + elif next_prev_tid > 0x7fffffff: + next_prev_tid -= 0x100000000 + + th_a_id = self.get_thread_id(machine_id, sample_pid, sample_tid) + comm_a = "Unknown_comm" + if self.session: + try: + proc = self.session.find_thread(sample_pid, sample_tid) + if proc: + comm_a_name = proc.comm() or "Unknown_comm" + if comm_a_name: + comm_a = comm_a_name + except TypeError: + pass + comm_a_id = self.get_comm_id(comm_a, th_a_id) + + th_b_id = self.get_thread_id(machine_id, next_prev_pid, next_prev_tid) + comm_b = "Unknown_comm" + if self.session: + try: + proc = self.session.find_thread(next_prev_pid, next_prev_tid) + if proc: + comm_b_name = proc.comm() or "Unknown_comm" + if comm_b_name: + comm_b = comm_b_name + except TypeError: + pass + comm_b_id = self.get_comm_id(comm_b, th_b_id) + + if out: + th_out_id = th_a_id + comm_out_id = comm_a_id + th_in_id = th_b_id + comm_in_id = comm_b_id + else: + th_out_id = th_b_id + comm_out_id = comm_b_id + th_in_id = th_a_id + comm_in_id = comm_a_id + + cs_id = self.next_id['context_switch'] + self.next_id['context_switch'] += 1 + time = getattr(event, "time", getattr(event, "sample_time", 0)) + cpu = getattr(event, "sample_cpu", 0) or 0 + if cpu > 0x7fffffff: + cpu -= 0x100000000 + + # 9 columns total: + fmt = "!hiQiqiQiiiqiqiqiqii" + value = struct.pack( + fmt, 9, 8, cs_id, 8, machine_id, 8, time, 4, cpu, + 8, th_out_id, 8, comm_out_id, 8, th_in_id, 8, comm_in_id, 4, flags + ) + self.files["context_switches"].write(value) + + def get_call_path_ids(self, call_path: typing.Any, machine_id: int = 0) -> tuple[int, int]: + """Add a perf.callchain as call_path rows, return its id and parent's.""" + parent_id = 0 + call_path_id = 0 + for node in call_path: + dso_name = node.dso or "Unknown_dso" + symbol_name = node.symbol or "Unknown_symbol" + dso_id = self.get_dso_id(dso_name, dso_name, "", machine_id) + symbol_id = self.get_symbol_id(dso_id, symbol_name, 0, 0) + parent_id = call_path_id + call_path_id = self.get_call_path_id(parent_id, symbol_id, node.ip) + return call_path_id, parent_id + + def process_call_return(self, cr: perf.call_return) -> None: + """Callback for processing call_return events.""" + machine_id = self.get_machine_id(getattr(cr, "machine_pid", 0) or 0) + thread_id = self.get_thread_id(machine_id, + cr.pid if cr.pid is not None else -1, + cr.tid if cr.tid is not None else -1) + comm_id = self.get_comm_id(cr.comm or "Unknown_comm", thread_id) + call_path_id, parent_call_path_id = self.get_call_path_ids(cr.call_path or [], machine_id) + + fmt = "!hiqiqiqiqiqiqiqiqiqiqiiiqiqiq" + # call.id and call.parent_id use the db_id of the perf module, as a call + # is given an id before it returns its parent cannot be numbered here. + value = struct.pack( + fmt, 14, 8, cr.db_id, 8, thread_id, 8, comm_id, 8, call_path_id, + 8, cr.call_time, 8, cr.return_time, 8, cr.branch_count, + 8, cr.call_ref, 8, cr.return_ref, + 8, parent_call_path_id, 4, cr.flags, 8, cr.parent_id, + 8, cr.insn_count, 8, cr.cyc_count + ) + self.files['call'].write(value) + + def process_event(self, sample: typing.Any) -> None: + """Callback for processing events.""" + + machine_db_id = getattr(sample, 'machine_pid', 0) + machine_id = self.get_machine_id(machine_db_id) + thread_id = self.get_thread_id(machine_id, sample.sample_pid, sample.sample_tid) + + comm = "Unknown_comm" + try: + if self.session is not None: + proc = self.session.find_thread(sample.sample_pid, sample.sample_tid) + if proc: + comm = proc.comm() or "Unknown_comm" + except TypeError: + pass + comm_id = self.get_comm_id(comm, thread_id) + + dso_id = self.get_dso_id( + sample.dso or "Unknown_dso", + sample.dso_long_name or "Unknown_dso_long", + sample.dso_bid or "", + machine_id + ) + + symbol_id = self.get_symbol_id( + dso_id, + sample.symbol or "Unknown_symbol", + sample.sym_start or 0, + sample.sym_end or 0 + ) + + call_path_id = 0 + if hasattr(sample, 'callchain') and sample.callchain: + parent_id = 0 + for node in reversed(sample.callchain): + node_dso = getattr(node, 'dso', None) or getattr(node, 'map', None) + node_symbol = getattr(node, 'symbol', None) or getattr(node, 'sym', None) + + dso_name = "Unknown_dso" + if node_dso: + dso_name = (node_dso if isinstance(node_dso, str) + else getattr(node_dso, 'name', "Unknown_dso") or "Unknown_dso") + + symbol_name = "Unknown_symbol" + if node_symbol: + symbol_name = (node_symbol if isinstance(node_symbol, str) + else getattr(node_symbol, 'name', "Unknown_symbol") + or "Unknown_symbol") + + node_dso_id = self.get_dso_id(dso_name, dso_name, "", machine_id) + node_symbol_id = self.get_symbol_id(node_dso_id, symbol_name, 0, 0) + + parent_id = self.get_call_path_id(parent_id, node_symbol_id, node.ip) + call_path_id = parent_id + + sample_id = self.next_id['sample'] + evsel_name = str(sample.evsel) + if evsel_name.startswith("evsel(") and evsel_name.endswith(")"): + evsel_name = evsel_name[6:-1] + self.write_sample(sample_id, + self.get_event_id(evsel_name), + machine_id, thread_id, comm_id, dso_id, symbol_id, sample, + call_path_id) + + event_name_str = evsel_name + if hasattr(sample, "raw_buf"): + if event_name_str == "ptwrite": + self.write_ptwrite(sample_id, sample.raw_buf) + elif event_name_str == "cbr": + self.write_cbr(sample_id, sample.raw_buf) + elif event_name_str == "mwait": + self.write_mwait(sample_id, sample.raw_buf) + elif event_name_str == "pwre": + self.write_pwre(sample_id, sample.raw_buf) + elif event_name_str == "exstop": + self.write_exstop(sample_id, sample.raw_buf) + elif event_name_str == "pwrx": + self.write_pwrx(sample_id, sample.raw_buf) + + self.next_id['sample'] += 1 + + def finalize(self) -> None: + """Copy files to database and add keys/views.""" + print("Copying to database...") + for symbol_id, (dso_id, sym_start, sym_end, binding, name) in self.symbol_rows.items(): + self.write_symbol(symbol_id, dso_id, sym_start, sym_end, binding, name) + for name, f in self.files.items(): + self.close_output_file(f) + + table_mapping = { + 'evsel': 'selected_events', + 'machine': 'machines', + 'thread': 'threads', + 'comm': 'comms', + 'comm_thread': 'comm_threads', + 'dso': 'dsos', + 'symbol': 'symbols', + 'branch_type': 'branch_types', + 'sample': 'samples', + 'call': 'calls', + 'call_path': 'call_paths', + 'context_switches': 'context_switches', + 'ptwrite': 'ptwrite', + 'cbr': 'cbr', + 'mwait': 'mwait', + 'pwre': 'pwre', + 'exstop': 'exstop', + 'pwrx': 'pwrx' + } + for name, f in self.files.items(): + table_name = table_mapping.get(name, name + "s") + self.copy_output_file(f.name, table_name) + + print("Removing intermediate files...") + for name, f in self.files.items(): + os.unlink(f.name) + shutil.rmtree(self.output_dir_name, ignore_errors=True) + + print("Adding primary keys") + self.do_query("ALTER TABLE selected_events ADD PRIMARY KEY (id)") + self.do_query("ALTER TABLE machines ADD PRIMARY KEY (id)") + self.do_query("ALTER TABLE threads ADD PRIMARY KEY (id)") + self.do_query("ALTER TABLE comms ADD PRIMARY KEY (id)") + self.do_query("ALTER TABLE comm_threads ADD PRIMARY KEY (id)") + self.do_query("ALTER TABLE dsos ADD PRIMARY KEY (id)") + self.do_query("ALTER TABLE symbols ADD PRIMARY KEY (id)") + self.do_query("ALTER TABLE branch_types ADD PRIMARY KEY (id)") + self.do_query("ALTER TABLE samples ADD PRIMARY KEY (id)") + self.do_query("ALTER TABLE call_paths ADD PRIMARY KEY (id)") + self.do_query("ALTER TABLE calls ADD PRIMARY KEY (id)") + self.do_query("ALTER TABLE ptwrite ADD PRIMARY KEY (id)") + self.do_query("ALTER TABLE cbr ADD PRIMARY KEY (id)") + self.do_query("ALTER TABLE mwait ADD PRIMARY KEY (id)") + self.do_query("ALTER TABLE pwre ADD PRIMARY KEY (id)") + self.do_query("ALTER TABLE exstop ADD PRIMARY KEY (id)") + self.do_query("ALTER TABLE pwrx ADD PRIMARY KEY (id)") + self.do_query("ALTER TABLE context_switches ADD PRIMARY KEY (id)") + + print("Adding foreign keys") + self.do_query('ALTER TABLE threads ' + 'ADD CONSTRAINT machinefk FOREIGN KEY (machine_id) REFERENCES machines (id),' + 'ADD CONSTRAINT processfk FOREIGN KEY (process_id) REFERENCES threads (id)') + self.do_query('ALTER TABLE comms ' + 'ADD CONSTRAINT threadfk FOREIGN KEY (c_thread_id) REFERENCES threads (id)') + self.do_query('ALTER TABLE comm_threads ' + 'ADD CONSTRAINT commfk FOREIGN KEY (comm_id) REFERENCES comms (id),' + 'ADD CONSTRAINT threadfk FOREIGN KEY (thread_id) REFERENCES threads (id)') + self.do_query('ALTER TABLE dsos ' + 'ADD CONSTRAINT machinefk FOREIGN KEY (machine_id) REFERENCES machines (id)') + self.do_query('ALTER TABLE symbols ' + 'ADD CONSTRAINT dsofk FOREIGN KEY (dso_id) REFERENCES dsos (id)') + self.do_query('ALTER TABLE samples ' + 'ADD CONSTRAINT evselfk FOREIGN KEY (evsel_id) ' + 'REFERENCES selected_events (id),' + 'ADD CONSTRAINT machinefk FOREIGN KEY (machine_id) REFERENCES machines (id),' + 'ADD CONSTRAINT threadfk FOREIGN KEY (thread_id) REFERENCES threads (id),' + 'ADD CONSTRAINT commfk FOREIGN KEY (comm_id) REFERENCES comms (id),' + 'ADD CONSTRAINT dsofk FOREIGN KEY (dso_id) REFERENCES dsos (id),' + 'ADD CONSTRAINT symbolfk FOREIGN KEY (symbol_id) REFERENCES symbols (id),' + 'ADD CONSTRAINT todsofk FOREIGN KEY (to_dso_id) REFERENCES dsos (id),' + 'ADD CONSTRAINT tosymbolfk FOREIGN KEY (to_symbol_id) ' + 'REFERENCES symbols (id)') + self.do_query('ALTER TABLE call_paths ' + 'ADD CONSTRAINT parentfk FOREIGN KEY (parent_id) REFERENCES call_paths (id),' + 'ADD CONSTRAINT symbolfk FOREIGN KEY (symbol_id) REFERENCES symbols (id)') + self.do_query('ALTER TABLE calls ' + 'ADD CONSTRAINT threadfk FOREIGN KEY (thread_id) REFERENCES threads (id),' + 'ADD CONSTRAINT commfk FOREIGN KEY (comm_id) REFERENCES comms (id),' + 'ADD CONSTRAINT call_pathfk FOREIGN KEY (call_path_id) ' + 'REFERENCES call_paths (id),' + 'ADD CONSTRAINT callfk FOREIGN KEY (call_id) REFERENCES samples (id),' + 'ADD CONSTRAINT returnfk FOREIGN KEY (return_id) REFERENCES samples (id),' + 'ADD CONSTRAINT parent_call_pathfk FOREIGN KEY (parent_call_path_id) ' + 'REFERENCES call_paths (id)') + # pcpid_idx, pid_idx, and comms.has_calls are required by + # exported-sql-viewer.py to filter the command dropdown to threads with + # call graphs and to expand parent/child call trees efficiently. + self.do_query('CREATE INDEX pcpid_idx ON calls (parent_call_path_id)') + self.do_query('CREATE INDEX pid_idx ON calls (parent_id)') + self.do_query('ALTER TABLE comms ADD has_calls boolean') + self.do_query('UPDATE comms SET has_calls = TRUE WHERE comms.id IN ' + '(SELECT DISTINCT comm_id FROM calls)') + self.do_query('ALTER TABLE context_switches ' + 'ADD CONSTRAINT machinefk FOREIGN KEY (machine_id) REFERENCES machines (id),' + 'ADD CONSTRAINT toutfk FOREIGN KEY (thread_out_id) REFERENCES threads (id),' + 'ADD CONSTRAINT tinfk FOREIGN KEY (thread_in_id) REFERENCES threads (id),' + 'ADD CONSTRAINT coutfk FOREIGN KEY (comm_out_id) REFERENCES comms (id),' + 'ADD CONSTRAINT cinfk FOREIGN KEY (comm_in_id) REFERENCES comms (id)') + self.do_query('ALTER TABLE ptwrite ' + 'ADD CONSTRAINT idfk FOREIGN KEY (id) REFERENCES samples (id)') + self.do_query('ALTER TABLE cbr ' + 'ADD CONSTRAINT idfk FOREIGN KEY (id) REFERENCES samples (id)') + self.do_query('ALTER TABLE mwait ' + 'ADD CONSTRAINT idfk FOREIGN KEY (id) REFERENCES samples (id)') + self.do_query('ALTER TABLE pwre ' + 'ADD CONSTRAINT idfk FOREIGN KEY (id) REFERENCES samples (id)') + self.do_query('ALTER TABLE exstop ' + 'ADD CONSTRAINT idfk FOREIGN KEY (id) REFERENCES samples (id)') + self.do_query('ALTER TABLE pwrx ' + 'ADD CONSTRAINT idfk FOREIGN KEY (id) REFERENCES samples (id)') + + print("Creating views...") + self.do_query( + "CREATE VIEW machines_view AS " + "SELECT id, pid, root_dir, " + "CASE WHEN id=0 THEN 'unknown' WHEN pid=-1 THEN 'host' ELSE 'guest' END " + "AS host_or_guest FROM machines" + ) + self.do_query( + "CREATE VIEW dsos_view AS " + "SELECT id, machine_id, " + "(SELECT host_or_guest FROM machines_view WHERE id = machine_id) AS host_or_guest, " + "short_name, long_name, build_id FROM dsos" + ) + self.do_query( + "CREATE VIEW symbols_view AS " + "SELECT id, name, (SELECT short_name FROM dsos WHERE id=dso_id) AS dso, " + "dso_id, sym_start, sym_end, " + "CASE WHEN binding=0 THEN 'local' WHEN binding=1 THEN 'global' ELSE 'weak' END " + "AS binding FROM symbols" + ) + self.do_query( + "CREATE VIEW threads_view AS " + "SELECT id, machine_id, " + "(SELECT host_or_guest FROM machines_view WHERE id = machine_id) AS host_or_guest, " + "process_id, pid, tid FROM threads" + ) + self.do_query( + "CREATE VIEW comm_threads_view AS " + "SELECT comm_id, (SELECT comm FROM comms WHERE id = comm_id) AS command, " + "thread_id, (SELECT pid FROM threads WHERE id = thread_id) AS pid, " + "(SELECT tid FROM threads WHERE id = thread_id) AS tid FROM comm_threads" + ) + self.do_query( + "CREATE VIEW call_paths_view AS " + "SELECT c.id, to_hex(c.ip) AS ip, c.symbol_id, " + "(SELECT name FROM symbols WHERE id = c.symbol_id) AS symbol, " + "(SELECT dso_id FROM symbols WHERE id = c.symbol_id) AS dso_id, " + "(SELECT dso FROM symbols_view WHERE id = c.symbol_id) AS dso_short_name, " + "c.parent_id, to_hex(p.ip) AS parent_ip, p.symbol_id AS parent_symbol_id, " + "(SELECT name FROM symbols WHERE id = p.symbol_id) AS parent_symbol, " + "(SELECT dso_id FROM symbols WHERE id = p.symbol_id) AS parent_dso_id, " + "(SELECT dso FROM symbols_view WHERE id = p.symbol_id) AS parent_dso_short_name " + "FROM call_paths c LEFT JOIN call_paths p ON p.id = c.parent_id" + ) + self.do_query( + "CREATE VIEW calls_view AS " + "SELECT calls.id, thread_id, " + "(SELECT pid FROM threads WHERE id = thread_id) AS pid, " + "(SELECT tid FROM threads WHERE id = thread_id) AS tid, " + "(SELECT comm FROM comms WHERE id = comm_id) AS command, " + "call_path_id, to_hex(ip) AS ip, symbol_id, " + "(SELECT name FROM symbols WHERE id = symbol_id) AS symbol, " + "call_time, return_time, return_time - call_time AS elapsed_time, " + "branch_count, insn_count, cyc_count, " + "CASE WHEN cyc_count=0 THEN CAST(0 AS FLOAT) " + "ELSE CAST(insn_count AS FLOAT) / cyc_count END AS IPC, " + "call_id, return_id, " + "CASE WHEN flags=0 THEN '' WHEN flags=1 THEN 'no call' " + "WHEN flags=2 THEN 'no return' WHEN flags=3 THEN 'no call/return' " + "WHEN flags IN (4, 6) THEN 'jmp' ELSE CAST(flags AS VARCHAR(6)) END AS flags, " + "parent_call_path_id, calls.parent_id " + "FROM calls INNER JOIN call_paths ON call_paths.id = call_path_id" + ) + self.do_query( + "CREATE VIEW samples_view AS " + "SELECT id, time, cpu, " + "(SELECT pid FROM threads WHERE id = thread_id) AS pid, " + "(SELECT tid FROM threads WHERE id = thread_id) AS tid, " + "(SELECT comm FROM comms WHERE id = comm_id) AS command, " + "(SELECT name FROM selected_events WHERE id = evsel_id) AS event, " + "to_hex(ip) AS ip_hex, " + "(SELECT name FROM symbols WHERE id = symbol_id) AS symbol, sym_offset, " + "(SELECT short_name FROM dsos WHERE id = dso_id) AS dso_short_name, " + "to_hex(to_ip) AS to_ip_hex, " + "(SELECT name FROM symbols WHERE id = to_symbol_id) AS to_symbol, to_sym_offset, " + "(SELECT short_name FROM dsos WHERE id = to_dso_id) AS to_dso_short_name, " + "(SELECT name FROM branch_types WHERE id = branch_type) AS branch_type_name, " + "in_tx, call_path_id, insn_count, cyc_count, " + "CASE WHEN cyc_count=0 THEN CAST(0 AS FLOAT) " + "ELSE CAST(insn_count AS FLOAT) / cyc_count END AS IPC, flags FROM samples" + ) + self.do_query( + "CREATE VIEW context_switches_view AS " + "SELECT context_switches.id, context_switches.machine_id, " + "context_switches.time, context_switches.cpu, " + "th_out.pid AS pid_out, th_out.tid AS tid_out, comm_out.comm AS comm_out, " + "th_in.pid AS pid_in, th_in.tid AS tid_in, comm_in.comm AS comm_in, " + "CASE WHEN flags=0 THEN 'in' WHEN flags=1 THEN 'out' " + "WHEN flags=3 THEN 'out preempt' ELSE CAST(flags AS VARCHAR(6)) END AS flags " + "FROM context_switches " + "INNER JOIN threads AS th_out ON th_out.id = context_switches.thread_out_id " + "INNER JOIN threads AS th_in ON th_in.id = context_switches.thread_in_id " + "INNER JOIN comms AS comm_out ON comm_out.id = context_switches.comm_out_id " + "INNER JOIN comms AS comm_in ON comm_in.id = context_switches.comm_in_id" + ) + self.do_query( + "CREATE VIEW ptwrite_view AS " + "SELECT ptwrite.id, time, cpu, to_hex(payload) AS payload_hex, " + "CASE WHEN exact_ip=FALSE THEN 'False' ELSE 'True' END AS exact_ip " + "FROM ptwrite INNER JOIN samples ON samples.id = ptwrite.id" + ) + self.do_query( + "CREATE VIEW cbr_view AS " + "SELECT cbr.id, time, cpu, cbr, mhz, percent " + "FROM cbr INNER JOIN samples ON samples.id = cbr.id" + ) + self.do_query( + "CREATE VIEW mwait_view AS " + "SELECT mwait.id, time, cpu, to_hex(hints) AS hints_hex, " + "to_hex(extensions) AS extensions_hex " + "FROM mwait INNER JOIN samples ON samples.id = mwait.id" + ) + self.do_query( + "CREATE VIEW pwre_view AS " + "SELECT pwre.id, time, cpu, cstate, subcstate, " + "CASE WHEN hw=FALSE THEN 'False' ELSE 'True' END AS hw " + "FROM pwre INNER JOIN samples ON samples.id = pwre.id" + ) + self.do_query( + "CREATE VIEW exstop_view AS " + "SELECT exstop.id, time, cpu, " + "CASE WHEN exact_ip=FALSE THEN 'False' ELSE 'True' END AS exact_ip " + "FROM exstop INNER JOIN samples ON samples.id = exstop.id" + ) + self.do_query( + "CREATE VIEW pwrx_view AS " + "SELECT pwrx.id, time, cpu, deepest_cstate, last_cstate, " + "CASE WHEN wake_reason=1 THEN 'Interrupt' " + "WHEN wake_reason=2 THEN 'Timer Deadline' " + "WHEN wake_reason=4 THEN 'Monitored Address' " + "WHEN wake_reason=8 THEN 'HW' WHEN wake_reason=16 THEN 'Other' " + "ELSE CAST ( wake_reason AS VARCHAR(2) ) END AS wake_reason " + "FROM pwrx INNER JOIN samples ON samples.id = pwrx.id" + ) + self.do_query( + "CREATE VIEW power_events_view AS " + "SELECT samples.id, samples.time, samples.cpu, selected_events.name AS event, " + "FORMAT('%6s', cbr.cbr) AS cbr, FORMAT('%6s', cbr.mhz) AS MHz, " + "FORMAT('%5s', cbr.percent) AS percent, to_hex(mwait.hints) AS hints_hex, " + "to_hex(mwait.extensions) AS extensions_hex, FORMAT('%3s', pwre.cstate) AS cstate, " + "FORMAT('%3s', pwre.subcstate) AS subcstate, " + "CASE WHEN pwre.hw=FALSE THEN 'False' WHEN pwre.hw=TRUE THEN 'True' " + "ELSE NULL END AS hw, " + "CASE WHEN exstop.exact_ip=FALSE THEN 'False' WHEN exstop.exact_ip=TRUE THEN 'True' " + "ELSE NULL END AS exact_ip, FORMAT('%3s', pwrx.deepest_cstate) AS deepest_cstate, " + "FORMAT('%3s', pwrx.last_cstate) AS last_cstate, " + "CASE WHEN pwrx.wake_reason=1 THEN 'Interrupt' " + "WHEN wake_reason=2 THEN 'Timer Deadline' " + "WHEN wake_reason=4 THEN 'Monitored Address' " + "WHEN wake_reason=8 THEN 'HW' WHEN wake_reason=16 THEN 'Other' " + "ELSE FORMAT('%2s', pwrx.wake_reason) END AS wake_reason " + "FROM cbr FULL JOIN mwait ON mwait.id = cbr.id " + "FULL JOIN pwre ON pwre.id = cbr.id FULL JOIN exstop ON exstop.id = cbr.id " + "FULL JOIN pwrx ON pwrx.id = cbr.id INNER JOIN samples ON samples.id = " + "coalesce(cbr.id, mwait.id, pwre.id, exstop.id, pwrx.id) " + "INNER JOIN selected_events ON selected_events.id = samples.evsel_id " + "ORDER BY samples.id" + ) + + +if __name__ == "__main__": + ap = argparse.ArgumentParser( + description="Export perf data to a postgresql database") + ap.add_argument("-i", "--input", default="perf.data", + help="Input file name") + ap.add_argument("-o", "--output", required=True, + help="Output database name") + ap.add_argument("--itrace", default=None, + help="itrace options, e.g. cr") + args = ap.parse_args() + + exporter = PostgresExporter(args.output) + + session = None + succeeded = False + caught_error = False + try: + exporter.setup_db() + session = perf.session(perf.data(args.input), + context_switch=exporter.process_context_switch, + sample=exporter.process_event, + call_return=exporter.process_call_return, + itrace=args.itrace) + exporter.session = session + try: + session.process_events() + finally: + # Break the C perf.session -> bound method -> exporter reference cycle. + exporter.session = None + exporter.finalize() + print(f"Successfully exported to {args.output}") + succeeded = True + except (OSError, RuntimeError, ValueError, KeyboardInterrupt) as e: + if not isinstance(e, (KeyboardInterrupt, SystemExit)): + import traceback + traceback.print_exc() + caught_error = True + finally: + exporter.session = None + for out_file in exporter.files.values(): + try: + if not out_file.closed: + out_file.close() + except OSError: + pass + exporter.disconnect() + if not succeeded: + if (getattr(exporter, 'created_output_dir', False) and + os.path.exists(exporter.output_dir_name)): + shutil.rmtree(exporter.output_dir_name, ignore_errors=True) + if caught_error: + sys.exit(1) diff --git a/tools/perf/python/export-to-sqlite.py b/tools/perf/python/export-to-sqlite.py new file mode 100755 index 000000000000..0a8087e51037 --- /dev/null +++ b/tools/perf/python/export-to-sqlite.py @@ -0,0 +1,946 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +""" +Export perf data to a sqlite3 database. + +This script has been ported to use the modern perf Python module and the +standard library sqlite3 module. It no longer requires PySide2 or QtSql +for exporting. + +Examples of using this script with Intel PT: + + $ perf record -e intel_pt//u ls + $ python export-to-sqlite.py -i perf.data -o pt_example + +To browse the database, sqlite3 can be used e.g. + + $ sqlite3 pt_example + sqlite> .header on + sqlite> select * from samples_view where id < 10; + sqlite> .mode column + sqlite> select * from samples_view where id < 10; + sqlite> .tables + sqlite> .schema samples_view + sqlite> .quit + +An example of using the database is provided by the script +exported-sql-viewer.py. Refer to that script for details. + +Ported from tools/perf/scripts/python/export-to-sqlite.py +""" + +from __future__ import annotations +import typing +import argparse +import os +import shutil +import sqlite3 +import struct +import sys +import tempfile +from typing import Dict, Optional +import perf + + +PERF_IP_FLAG_BRANCH = 1 << 0 +PERF_IP_FLAG_CALL = 1 << 1 +PERF_IP_FLAG_RETURN = 1 << 2 +PERF_IP_FLAG_CONDITIONAL = 1 << 3 +PERF_IP_FLAG_SYSCALLRET = 1 << 4 +PERF_IP_FLAG_ASYNC = 1 << 5 +PERF_IP_FLAG_INTERRUPT = 1 << 6 +PERF_IP_FLAG_TX_ABORT = 1 << 7 +PERF_IP_FLAG_TRACE_BEGIN = 1 << 8 +PERF_IP_FLAG_TRACE_END = 1 << 9 +PERF_IP_FLAG_IN_TX = 1 << 10 +PERF_IP_FLAG_VMENTRY = 1 << 11 +PERF_IP_FLAG_VMEXIT = 1 << 12 + +BRANCH_TYPES = [ + (0, "no branch"), + (PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL, "call"), + (PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_RETURN, "return"), + (PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CONDITIONAL, "conditional jump"), + (PERF_IP_FLAG_BRANCH, "unconditional jump"), + (PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL | PERF_IP_FLAG_INTERRUPT, "software interrupt"), + (PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_RETURN | PERF_IP_FLAG_INTERRUPT, "return from interrupt"), + (PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL | PERF_IP_FLAG_SYSCALLRET, "system call"), + (PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_RETURN | PERF_IP_FLAG_SYSCALLRET, + "return from system call"), + (PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_ASYNC, "asynchronous branch"), + (PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL | PERF_IP_FLAG_ASYNC | PERF_IP_FLAG_INTERRUPT, + "hardware interrupt"), + (PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_TX_ABORT, "transaction abort"), + (PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_TRACE_BEGIN, "trace begin"), + (PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_TRACE_END, "trace end"), + (PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL | PERF_IP_FLAG_VMENTRY, "vm entry"), + (PERF_IP_FLAG_BRANCH | PERF_IP_FLAG_CALL | PERF_IP_FLAG_VMEXIT, "vm exit"), +] +for _b_type, _b_name in list(BRANCH_TYPES): + if (_b_type == PERF_IP_FLAG_BRANCH or + (_b_type & (PERF_IP_FLAG_TRACE_BEGIN | PERF_IP_FLAG_TRACE_END))): + continue + BRANCH_TYPES.append((_b_type | PERF_IP_FLAG_TRACE_BEGIN, f"trace begin / {_b_name}")) + BRANCH_TYPES.append((_b_type | PERF_IP_FLAG_TRACE_END, f"{_b_name} / trace end")) + + +class DatabaseExporter: + """Handles database connection and exporting of perf events.""" + + def __init__(self, db_path: str): + self.con = sqlite3.connect(db_path) + self.con.execute("PRAGMA journal_mode = MEMORY") + self.session: Optional[perf.session] = None + self.sample_count = 0 + + # Caches and counters grouped to reduce instance attributes + self.caches: Dict[str, dict] = { + 'machines': {}, + 'threads': {}, + 'comms': {}, + 'dsos': {}, + 'symbols': {}, + 'events': {}, + 'branch_types': {}, + 'call_paths': {} + } + + self.next_id = { + 'machine': 1, + 'thread': 1, + 'comm': 1, + 'dso': 1, + 'symbol': 1, + 'event': 1, + 'branch_type': 1, + 'call_path': 1 + } + + self.create_tables() + + def create_tables(self) -> None: + """Create database tables.""" + self.con.execute(""" + CREATE TABLE IF NOT EXISTS selected_events ( + id INTEGER NOT NULL PRIMARY KEY, + name VARCHAR(80)) + """) + self.con.execute(""" + CREATE TABLE IF NOT EXISTS machines ( + id INTEGER NOT NULL PRIMARY KEY, + pid INTEGER, + root_dir VARCHAR(4096)) + """) + self.con.execute(""" + CREATE TABLE IF NOT EXISTS threads ( + id INTEGER NOT NULL PRIMARY KEY, + machine_id BIGINT, + process_id BIGINT, + pid INTEGER, + tid INTEGER) + """) + self.con.execute(""" + CREATE TABLE IF NOT EXISTS comms ( + id INTEGER NOT NULL PRIMARY KEY, + comm VARCHAR(16), + c_thread_id BIGINT, + c_time BIGINT, + exec_flag BOOLEAN) + """) + self.con.execute(""" + CREATE TABLE IF NOT EXISTS comm_threads ( + id INTEGER NOT NULL PRIMARY KEY, + comm_id BIGINT, + thread_id BIGINT) + """) + self.con.execute(""" + CREATE TABLE IF NOT EXISTS dsos ( + id INTEGER NOT NULL PRIMARY KEY, + machine_id BIGINT, + short_name VARCHAR(256), + long_name VARCHAR(4096), + build_id VARCHAR(64)) + """) + self.con.execute(""" + CREATE TABLE IF NOT EXISTS symbols ( + id INTEGER NOT NULL PRIMARY KEY, + dso_id BIGINT, + sym_start BIGINT, + sym_end BIGINT, + binding INTEGER, + name VARCHAR(2048)) + """) + self.con.execute(""" + CREATE TABLE IF NOT EXISTS branch_types ( + id INTEGER NOT NULL PRIMARY KEY, + name VARCHAR(80)) + """) + self.con.execute(""" + CREATE TABLE IF NOT EXISTS samples ( + id INTEGER NOT NULL PRIMARY KEY, + evsel_id BIGINT, + machine_id BIGINT, + thread_id BIGINT, + comm_id BIGINT, + dso_id BIGINT, + symbol_id BIGINT, + sym_offset BIGINT, + ip BIGINT, + time BIGINT, + cpu INTEGER, + to_dso_id BIGINT, + to_symbol_id BIGINT, + to_sym_offset BIGINT, + to_ip BIGINT, + period BIGINT, + weight BIGINT, + transaction_ BIGINT, + data_src BIGINT, + branch_type INTEGER, + in_tx BOOLEAN, + call_path_id BIGINT, + insn_count BIGINT, + cyc_count BIGINT, + flags INTEGER) + """) + self.con.execute(''' + CREATE TABLE IF NOT EXISTS calls ( + id INTEGER NOT NULL PRIMARY KEY, + thread_id BIGINT, + comm_id BIGINT, + call_path_id BIGINT, + call_time BIGINT, + return_time BIGINT, + branch_count BIGINT, + call_id BIGINT, + return_id BIGINT, + parent_call_path_id BIGINT, + flags INTEGER, + parent_id BIGINT, + insn_count BIGINT, + cyc_count BIGINT + ) + ''') + self.con.execute(''' + CREATE TABLE IF NOT EXISTS call_paths ( + id INTEGER NOT NULL PRIMARY KEY, + parent_id BIGINT, + symbol_id BIGINT, + ip BIGINT + ) + ''') + + self.con.execute(''' + CREATE TABLE IF NOT EXISTS cbr ( + id integer primary key, + cbr integer, + mhz integer, + percent integer + ) + ''') + self.con.execute(''' + CREATE TABLE IF NOT EXISTS mwait ( + id integer primary key, + hints integer, + extensions integer + ) + ''') + + self.con.execute(''' + CREATE TABLE IF NOT EXISTS context_switches ( + id integer primary key, + machine_id bigint, + time bigint, + cpu integer, + thread_out_id bigint, + comm_out_id bigint, + thread_in_id bigint, + comm_in_id bigint, + flags integer + ) + ''') + self.con.execute( + "CREATE TABLE IF NOT EXISTS ptwrite (" + "id integer primary key, payload integer, exact_ip integer" + ")" + ) + self.con.execute( + "CREATE TABLE IF NOT EXISTS pwre (" + "id integer primary key, hw integer, cstate integer, subcstate integer, " + "hw_name text, cstate_name text" + ")" + ) + self.con.execute( + "CREATE TABLE IF NOT EXISTS exstop (" + "id integer primary key, exact_ip integer" + ")" + ) + self.con.execute( + "CREATE TABLE IF NOT EXISTS pwrx (" + "id integer primary key, deepest_cstate integer, last_cstate integer, " + "wake_reason integer" + ")" + ) + + self.con.execute( + "CREATE VIEW IF NOT EXISTS machines_view AS " + "SELECT id, pid, root_dir, " + "CASE WHEN id=0 THEN 'unknown' WHEN pid=-1 THEN 'host' ELSE 'guest' END " + "AS host_or_guest FROM machines" + ) + self.con.execute( + "CREATE VIEW IF NOT EXISTS dsos_view AS " + "SELECT id, machine_id, " + "(SELECT host_or_guest FROM machines_view WHERE id = machine_id) AS host_or_guest, " + "short_name, long_name, build_id FROM dsos" + ) + self.con.execute( + "CREATE VIEW IF NOT EXISTS symbols_view AS " + "SELECT id, name, (SELECT short_name FROM dsos WHERE id=dso_id) AS dso, " + "dso_id, sym_start, sym_end, " + "CASE WHEN binding=0 THEN 'local' WHEN binding=1 THEN 'global' ELSE 'weak' END " + "AS binding FROM symbols" + ) + self.con.execute( + "CREATE VIEW IF NOT EXISTS threads_view AS " + "SELECT id, machine_id, " + "(SELECT host_or_guest FROM machines_view WHERE id = machine_id) AS host_or_guest, " + "process_id, pid, tid FROM threads" + ) + self.con.execute( + "CREATE VIEW IF NOT EXISTS comm_threads_view AS " + "SELECT comm_id, (SELECT comm FROM comms WHERE id = comm_id) AS command, " + "thread_id, (SELECT pid FROM threads WHERE id = thread_id) AS pid, " + "(SELECT tid FROM threads WHERE id = thread_id) AS tid FROM comm_threads" + ) + self.con.execute( + "CREATE VIEW IF NOT EXISTS call_paths_view AS " + "SELECT c.id, printf('%x', c.ip) AS ip, c.symbol_id, " + "(SELECT name FROM symbols WHERE id = c.symbol_id) AS symbol, " + "(SELECT dso_id FROM symbols WHERE id = c.symbol_id) AS dso_id, " + "(SELECT dso FROM symbols_view WHERE id = c.symbol_id) AS dso_short_name, " + "c.parent_id, printf('%x', p.ip) AS parent_ip, p.symbol_id AS parent_symbol_id, " + "(SELECT name FROM symbols WHERE id = p.symbol_id) AS parent_symbol, " + "(SELECT dso_id FROM symbols WHERE id = p.symbol_id) AS parent_dso_id, " + "(SELECT dso FROM symbols_view WHERE id = p.symbol_id) AS parent_dso_short_name " + "FROM call_paths c LEFT JOIN call_paths p ON p.id = c.parent_id" + ) + self.con.execute( + "CREATE VIEW IF NOT EXISTS calls_view AS " + "SELECT calls.id, thread_id, " + "(SELECT pid FROM threads WHERE id = thread_id) AS pid, " + "(SELECT tid FROM threads WHERE id = thread_id) AS tid, " + "(SELECT comm FROM comms WHERE id = comm_id) AS command, " + "call_path_id, printf('%x', ip) AS ip, symbol_id, " + "(SELECT name FROM symbols WHERE id = symbol_id) AS symbol, " + "call_time, return_time, return_time - call_time AS elapsed_time, " + "branch_count, insn_count, cyc_count, " + "CASE WHEN cyc_count=0 THEN CAST(0 AS FLOAT) " + "ELSE CAST(insn_count AS FLOAT) / cyc_count END AS IPC, " + "call_id, return_id, " + "CASE WHEN flags=0 THEN '' WHEN flags=1 THEN 'no call' " + "WHEN flags=2 THEN 'no return' WHEN flags=3 THEN 'no call/return' " + "WHEN flags IN (4, 6) THEN 'jmp' ELSE flags END AS flags, " + "parent_call_path_id, calls.parent_id " + "FROM calls INNER JOIN call_paths ON call_paths.id = call_path_id" + ) + self.con.execute( + "CREATE VIEW IF NOT EXISTS samples_view AS " + "SELECT id, time, cpu, " + "(SELECT pid FROM threads WHERE id = thread_id) AS pid, " + "(SELECT tid FROM threads WHERE id = thread_id) AS tid, " + "(SELECT comm FROM comms WHERE id = comm_id) AS command, " + "(SELECT name FROM selected_events WHERE id = evsel_id) AS event, " + "printf('%x', ip) AS ip_hex, " + "(SELECT name FROM symbols WHERE id = symbol_id) AS symbol, sym_offset, " + "(SELECT short_name FROM dsos WHERE id = dso_id) AS dso_short_name, " + "printf('%x', to_ip) AS to_ip_hex, " + "(SELECT name FROM symbols WHERE id = to_symbol_id) AS to_symbol, to_sym_offset, " + "(SELECT short_name FROM dsos WHERE id = to_dso_id) AS to_dso_short_name, " + "(SELECT name FROM branch_types WHERE id = branch_type) AS branch_type_name, " + "in_tx, call_path_id, insn_count, cyc_count, " + "CASE WHEN cyc_count=0 THEN CAST(0 AS FLOAT) " + "ELSE CAST(insn_count AS FLOAT) / cyc_count END AS IPC, flags FROM samples" + ) + self.con.execute( + "CREATE VIEW IF NOT EXISTS ptwrite_view AS " + "SELECT ptwrite.id, time, cpu, printf('%x', payload) AS payload_hex, " + "CASE WHEN exact_ip=0 THEN 'False' ELSE 'True' END AS exact_ip " + "FROM ptwrite INNER JOIN samples ON samples.id = ptwrite.id" + ) + self.con.execute( + "CREATE VIEW IF NOT EXISTS cbr_view AS " + "SELECT cbr.id, time, cpu, cbr, mhz, percent " + "FROM cbr INNER JOIN samples ON samples.id = cbr.id" + ) + self.con.execute( + "CREATE VIEW IF NOT EXISTS mwait_view AS " + "SELECT mwait.id, time, cpu, printf('%x', hints) AS hints_hex, " + "printf('%x', extensions) AS extensions_hex " + "FROM mwait INNER JOIN samples ON samples.id = mwait.id" + ) + self.con.execute( + "CREATE VIEW IF NOT EXISTS pwre_view AS " + "SELECT pwre.id, time, cpu, cstate, subcstate, " + "CASE WHEN hw=0 THEN 'False' ELSE 'True' END AS hw " + "FROM pwre INNER JOIN samples ON samples.id = pwre.id" + ) + self.con.execute( + "CREATE VIEW IF NOT EXISTS exstop_view AS " + "SELECT exstop.id, time, cpu, " + "CASE WHEN exact_ip=0 THEN 'False' ELSE 'True' END AS exact_ip " + "FROM exstop INNER JOIN samples ON samples.id = exstop.id" + ) + self.con.execute( + "CREATE VIEW IF NOT EXISTS pwrx_view AS " + "SELECT pwrx.id, time, cpu, deepest_cstate, last_cstate, " + "CASE WHEN wake_reason=1 THEN 'Interrupt' " + "WHEN wake_reason=2 THEN 'Timer Deadline' " + "WHEN wake_reason=4 THEN 'Monitored Address' " + "WHEN wake_reason=8 THEN 'HW' WHEN wake_reason=16 THEN 'Other' " + "ELSE wake_reason END AS wake_reason " + "FROM pwrx INNER JOIN samples ON samples.id = pwrx.id" + ) + self.con.execute( + "CREATE VIEW IF NOT EXISTS power_events_view AS " + "SELECT samples.id, time, cpu, selected_events.name AS event, " + "CASE WHEN selected_events.name='cbr' THEN " + "(SELECT cbr FROM cbr WHERE cbr.id = samples.id) ELSE \"\" END AS cbr, " + "CASE WHEN selected_events.name='cbr' THEN " + "(SELECT mhz FROM cbr WHERE cbr.id = samples.id) ELSE \"\" END AS mhz, " + "CASE WHEN selected_events.name='cbr' THEN " + "(SELECT percent FROM cbr WHERE cbr.id = samples.id) ELSE \"\" END AS percent, " + "CASE WHEN selected_events.name='mwait' THEN " + "(SELECT printf('%x', hints) FROM mwait WHERE mwait.id = samples.id) " + "ELSE \"\" END AS hints_hex, " + "CASE WHEN selected_events.name='mwait' THEN " + "(SELECT printf('%x', extensions) FROM mwait WHERE mwait.id = samples.id) " + "ELSE \"\" END AS extensions_hex, " + "CASE WHEN selected_events.name='pwre' THEN " + "(SELECT cstate FROM pwre WHERE pwre.id = samples.id) ELSE \"\" END AS cstate, " + "CASE WHEN selected_events.name='pwre' THEN " + "(SELECT subcstate FROM pwre WHERE pwre.id = samples.id) ELSE \"\" END AS subcstate, " + "CASE WHEN selected_events.name='pwre' THEN " + "(SELECT hw FROM pwre WHERE pwre.id = samples.id) ELSE \"\" END AS hw, " + "CASE WHEN selected_events.name='exstop' THEN " + "(SELECT exact_ip FROM exstop WHERE exstop.id = samples.id) " + "ELSE \"\" END AS exact_ip, " + "CASE WHEN selected_events.name='pwrx' THEN " + "(SELECT deepest_cstate FROM pwrx WHERE pwrx.id = samples.id) " + "ELSE \"\" END AS deepest_cstate, " + "CASE WHEN selected_events.name='pwrx' THEN " + "(SELECT last_cstate FROM pwrx WHERE pwrx.id = samples.id) " + "ELSE \"\" END AS last_cstate, " + "CASE WHEN selected_events.name='pwrx' THEN " + "(SELECT wake_reason FROM pwrx WHERE pwrx.id = samples.id) " + "ELSE \"\" END AS wake_reason " + "FROM samples INNER JOIN selected_events ON selected_events.id = evsel_id " + "WHERE selected_events.name IN ('cbr','mwait','pwre','exstop','pwrx')" + ) + self.con.execute( + "CREATE VIEW IF NOT EXISTS context_switches_view AS " + "SELECT context_switches.id, context_switches.machine_id, " + "context_switches.time, context_switches.cpu, " + "th_out.pid AS pid_out, th_out.tid AS tid_out, comm_out.comm AS comm_out, " + "th_in.pid AS pid_in, th_in.tid AS tid_in, comm_in.comm AS comm_in, " + "CASE WHEN flags=0 THEN 'in' WHEN flags=1 THEN 'out' " + "WHEN flags=3 THEN 'out preempt' ELSE flags END AS flags " + "FROM context_switches " + "INNER JOIN threads AS th_out ON th_out.id = context_switches.thread_out_id " + "INNER JOIN threads AS th_in ON th_in.id = context_switches.thread_in_id " + "INNER JOIN comms AS comm_out ON comm_out.id = context_switches.comm_out_id " + "INNER JOIN comms AS comm_in ON comm_in.id = context_switches.comm_in_id" + ) + + # id == 0 means unknown. It is easier to create records for them than + # replace the zeroes with NULLs + self.con.execute("INSERT OR IGNORE INTO selected_events VALUES (0, 'unknown')") + self.con.execute("INSERT OR IGNORE INTO machines VALUES (0, 0, 'unknown')") + self.con.execute("INSERT OR IGNORE INTO threads VALUES (0, 0, 0, -1, -1)") + self.con.execute("INSERT OR IGNORE INTO comms VALUES (0, 'unknown', 0, 0, 0)") + self.con.execute("INSERT OR IGNORE INTO dsos VALUES (0, 0, 'unknown', 'unknown', '')") + self.con.execute("INSERT OR IGNORE INTO call_paths VALUES (0, 0, 0, 0)") + self.con.execute("INSERT OR IGNORE INTO symbols VALUES (0, 0, 0, 0, 0, 'unknown')") + for b_type, b_name in BRANCH_TYPES: + self.con.execute("INSERT OR IGNORE INTO branch_types VALUES (?, ?)", (b_type, b_name)) + self.caches['branch_types'][b_type] = b_name + + self.caches['events']['unknown'] = 0 + self.caches['threads'][(0, -1, -1)] = 0 + self.caches['threads'][(0, 0, 0)] = 0 + self.caches['comms'][('unknown', 0)] = 0 + self.caches['dsos'][(0, 'unknown', 'unknown', '')] = 0 + self.caches['symbols'][(0, 'unknown')] = (0, 0, 0) + # Initialize comm_threads mapping + self.comm_threads_cache: set[tuple[int, int]] = set() + self.next_comm_thread_id = 1 + + def _exec(self, sql: str, params: tuple[typing.Any, ...] = ()) -> sqlite3.Cursor: + """Execute SQL statement converting unsigned 64-bit ints to signed 64-bit. + + Python's sqlite3 module only binds signed 64-bit integers (int64_t) and + raises OverflowError for unsigned 64-bit kernel addresses (>= 2^63). + Convert values in [2^63, 2^64) into two's-complement signed 64-bit ints. + """ + conv_params = tuple( + p - 0x10000000000000000 if isinstance(p, int) and p >= 0x8000000000000000 else p + for p in params + ) + return self.con.execute(sql, conv_params) + + def get_machine_id(self, machine_pid: Optional[int]) -> int: + """Get or create machine ID.""" + if machine_pid is None or machine_pid <= 0 or machine_pid > 0x7fffffff: + machine_pid = -1 + if machine_pid in self.caches['machines']: + return self.caches['machines'][machine_pid] + machine_id = self.next_id['machine'] + self.next_id['machine'] += 1 + self._exec("INSERT INTO machines VALUES (?, ?, ?)", + (machine_id, machine_pid, "")) + self.caches['machines'][machine_pid] = machine_id + return machine_id + + def get_event_id(self, name: str) -> int: + """Get or create event ID.""" + if name in self.caches['events']: + return self.caches['events'][name] + event_id = self.next_id['event'] + self._exec("INSERT INTO selected_events VALUES (?, ?)", + (event_id, name)) + self.caches['events'][name] = event_id + self.next_id['event'] += 1 + return event_id + + def get_thread_id(self, machine_id: int, pid: Optional[int], tid: Optional[int]) -> int: + """Get or create thread ID.""" + if pid is None: + pid = -1 + elif pid > 0x7fffffff: + pid -= 0x100000000 + if tid is None: + tid = -1 + elif tid > 0x7fffffff: + tid -= 0x100000000 + key = (machine_id, pid, tid) + if key in self.caches['threads']: + return self.caches['threads'][key] + thread_id = self.next_id['thread'] + self.next_id['thread'] += 1 + self.caches['threads'][key] = thread_id + process_id = thread_id if pid == tid else self.get_thread_id(machine_id, pid, pid) + self._exec("INSERT INTO threads VALUES (?, ?, ?, ?, ?)", + (thread_id, machine_id, process_id, pid, tid)) + return thread_id + + def get_comm_id(self, comm: str, thread_id: int) -> int: + """Get or create comm ID.""" + key = (comm, thread_id) + if key not in self.caches['comms']: + comm_id = self.next_id['comm'] + self._exec("INSERT INTO comms VALUES (?, ?, ?, ?, ?)", + (comm_id, comm, thread_id, 0, 0)) + self.caches['comms'][key] = comm_id + self.next_id['comm'] += 1 + comm_id = self.caches['comms'][key] + mapping_key = (comm_id, thread_id) + if mapping_key not in self.comm_threads_cache: + comm_thread_id = self.next_comm_thread_id + self._exec("INSERT INTO comm_threads VALUES (?, ?, ?)", + (comm_thread_id, comm_id, thread_id)) + self.comm_threads_cache.add(mapping_key) + self.next_comm_thread_id += 1 + return comm_id + + def get_dso_id(self, short_name: str, long_name: str, + build_id: str, machine_id: int = 0) -> int: + """Get or create DSO ID.""" + key = (machine_id, short_name, long_name, build_id) + if key in self.caches['dsos']: + return self.caches['dsos'][key] + short_key = (machine_id, short_name) + if not build_id and short_key in self.caches['dsos']: + return self.caches['dsos'][short_key] + dso_id = self.next_id['dso'] + self._exec("INSERT INTO dsos VALUES (?, ?, ?, ?, ?)", + (dso_id, machine_id, short_name, long_name, build_id)) + self.caches['dsos'][key] = dso_id + self.caches['dsos'][short_key] = dso_id + self.next_id['dso'] += 1 + return dso_id + + def get_symbol_id(self, dso_id: int, name: str, start: int, + end: int) -> int: + """Get or create symbol ID.""" + key = (dso_id, name) + if key in self.caches['symbols']: + symbol_id, cached_start, cached_end = self.caches['symbols'][key] + if cached_start == 0 and cached_end == 0 and (start != 0 or end != 0): + self._exec("UPDATE symbols SET sym_start = ?, sym_end = ? WHERE id = ?", + (start, end, symbol_id)) + self.caches['symbols'][key] = (symbol_id, start, end) + return symbol_id + symbol_id = self.next_id['symbol'] + self._exec("INSERT INTO symbols VALUES (?, ?, ?, ?, ?, ?)", + (symbol_id, dso_id, start, end, 0, name)) + self.caches['symbols'][key] = (symbol_id, start, end) + self.next_id['symbol'] += 1 + return symbol_id + + def get_call_path_id(self, parent_id: int, symbol_id: int, + ip: int) -> int: + """Get or create call path ID.""" + key = (parent_id, symbol_id, ip) + if key in self.caches['call_paths']: + return self.caches['call_paths'][key] + call_path_id = self.next_id['call_path'] + self._exec("INSERT INTO call_paths VALUES (?, ?, ?, ?)", + (call_path_id, parent_id, symbol_id, ip)) + self.caches['call_paths'][key] = call_path_id + self.next_id['call_path'] += 1 + return call_path_id + + def process_event(self, sample: perf.sample_event) -> None: + """Callback for processing events.""" + + machine_pid = getattr(sample, 'machine_pid', 0) + machine_id = self.get_machine_id(machine_pid) + thread_id = self.get_thread_id(machine_id, sample.sample_pid, sample.sample_tid) + + comm = "Unknown_comm" + try: + if self.session is not None: + proc = self.session.find_thread(sample.sample_pid, sample.sample_tid) + if proc: + comm_name = proc.comm() + if comm_name: + comm = comm_name + except TypeError: + pass + comm_id = self.get_comm_id(comm, thread_id) + + build_id = sample.dso_bid or "" + dso_id = self.get_dso_id( + sample.dso or "Unknown_dso", + sample.dso_long_name or "Unknown_dso_long", + build_id, + machine_id + ) + + symbol_id = self.get_symbol_id( + dso_id, + sample.symbol or "Unknown_symbol", + sample.sym_start or 0, + sample.sym_end or 0 + ) + + # Handle callchain + call_path_id = 0 + if hasattr(sample, 'callchain') and sample.callchain: + parent_id = 0 + for node in reversed(sample.callchain): + dso_name = node.dso or "Unknown_dso" + symbol_name = node.symbol or "Unknown_symbol" + + node_dso_id = self.get_dso_id(dso_name, dso_name, "", machine_id) + node_symbol_id = self.get_symbol_id(node_dso_id, symbol_name, 0, 0) + + parent_id = self.get_call_path_id(parent_id, node_symbol_id, node.ip) + call_path_id = parent_id + else: + call_path_id = 0 + + # Insert sample + event_name_str = str(sample.evsel) + if event_name_str.startswith("evsel(") and event_name_str.endswith(")"): + event_name_str = event_name_str[6:-1] + cursor = self._exec(""" + INSERT INTO samples VALUES ( + NULL, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ? + ) + """, ( + self.get_event_id(event_name_str), + machine_id, thread_id, comm_id, + dso_id, symbol_id, + getattr(sample, 'sym_offset', 0) or 0, + sample.sample_ip, sample.sample_time, sample.sample_cpu, + self.get_dso_id(sample.addr_dso or "Unknown_dso", + sample.addr_dso or "Unknown_dso_long", "", machine_id), + self.get_symbol_id( + self.get_dso_id(sample.addr_dso or "Unknown_dso", + sample.addr_dso or "Unknown_dso_long", "", machine_id), + sample.addr_symbol or "Unknown_symbol", 0, 0 + ), + sample.addr_sym_offset or 0, + sample.sample_addr or 0, + sample.sample_period or 0, + sample.sample_weight or 0, + sample.transaction or 0, + sample.sample_data_src, + sample.branch_type or 0, + sample.in_tx or 0, + call_path_id, + sample.sample_insn_count, + sample.sample_cyc_count, + getattr(sample, "flags", 0) # flags + )) + sample_id = cursor.lastrowid + + # Handle Intel PT specific raw payloads mathematically equivalent to + # legacy synth_data unpacking + + if event_name_str == "ptwrite" and hasattr(sample, "raw_buf"): + try: + flags, payload = struct.unpack_from("<IQ", sample.raw_buf) + self._exec("INSERT INTO ptwrite VALUES (?, ?, ?)", + (sample_id, payload, flags & 1)) + except struct.error: + pass + + elif event_name_str == "cbr" and hasattr(sample, "raw_buf"): + try: + data = struct.unpack_from("<BBBBII", sample.raw_buf) + cbr_val, max_nonturbo, freq = data[0], data[2], data[4] + MHz = (freq + 500) // 1000 + percent = ((cbr_val * 1000 // max_nonturbo) + 5) // 10 if max_nonturbo else 0 + self._exec("INSERT INTO cbr VALUES (?, ?, ?, ?)", + (sample_id, cbr_val, MHz, percent)) + except struct.error: + pass + + elif event_name_str == "mwait" and hasattr(sample, "raw_buf"): + try: + flags, payload = struct.unpack_from("<IQ", sample.raw_buf) + hints = payload & 0xff + extensions = (payload >> 32) & 0x3 + self._exec("INSERT INTO mwait VALUES (?, ?, ?)", + (sample_id, hints, extensions)) + except struct.error: + pass + + elif event_name_str == "pwre" and hasattr(sample, "raw_buf"): + try: + flags, payload = struct.unpack_from("<IQ", sample.raw_buf) + hw = (payload >> 7) & 1 + cstate = (payload >> 12) & 0xf + subcstate = (payload >> 8) & 0xf + self._exec("INSERT INTO pwre (id, cstate, subcstate, hw) VALUES (?, ?, ?, ?)", + (sample_id, cstate, subcstate, hw)) + except struct.error: + pass + + elif event_name_str == "exstop" and hasattr(sample, "raw_buf"): + try: + flags = struct.unpack_from("<I", sample.raw_buf)[0] + self._exec("INSERT INTO exstop VALUES (?, ?)", + (sample_id, flags & 1)) + except struct.error: + pass + + elif event_name_str == "pwrx" and hasattr(sample, "raw_buf"): + try: + flags, payload = struct.unpack_from("<IQ", sample.raw_buf) + deepest_cstate = payload & 0xf + last_cstate = (payload >> 4) & 0xf + wake_reason = (payload >> 8) & 0xf + self._exec("INSERT INTO pwrx VALUES (?, ?, ?, ?)", + (sample_id, deepest_cstate, last_cstate, wake_reason)) + except struct.error: + pass + + self.sample_count += 1 + if self.sample_count % 10000 == 0: + self.commit() + + def get_call_path_ids(self, call_path: typing.Any, machine_id: int = 0) -> tuple[int, int]: + """Add a perf.callchain as call_paths rows, return its id and parent's.""" + parent_id = 0 + call_path_id = 0 + for node in call_path: + dso_name = node.dso or "Unknown_dso" + symbol_name = node.symbol or "Unknown_symbol" + dso_id = self.get_dso_id(dso_name, dso_name, "", machine_id) + symbol_id = self.get_symbol_id(dso_id, symbol_name, 0, 0) + parent_id = call_path_id + call_path_id = self.get_call_path_id(parent_id, symbol_id, node.ip) + return call_path_id, parent_id + + def process_call_return(self, cr: perf.call_return) -> None: + """Callback for processing call_return events.""" + machine_id = self.get_machine_id(getattr(cr, "machine_pid", None)) + thread_id = self.get_thread_id(machine_id, cr.pid, cr.tid) + comm_id = self.get_comm_id(cr.comm or "Unknown_comm", thread_id) + call_path_id, parent_call_path_id = self.get_call_path_ids(cr.call_path or [], machine_id) + # calls.id and calls.parent_id use the db_id of the perf module, as a + # call is given an id before it returns its parent cannot be numbered + # here. + self._exec( + "INSERT INTO calls VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)", + (cr.db_id, thread_id, comm_id, call_path_id, + cr.call_time, cr.return_time, + cr.branch_count, cr.call_ref, + cr.return_ref, parent_call_path_id, + cr.flags, cr.parent_id, + cr.insn_count, cr.cyc_count) + ) + + def process_context_switch(self, event: typing.Any) -> None: + """Callback for processing context switch events.""" + misc = getattr(event, 'misc', 0) + out = bool(misc & (1 << 13)) # PERF_RECORD_MISC_SWITCH_OUT + out_preempt = bool(misc & (1 << 14)) # PERF_RECORD_MISC_SWITCH_OUT_PREEMPT + flags = (1 if out else 0) | ((1 if out_preempt else 0) << 1) + machine_pid = getattr(event, 'machine_pid', None) + machine_id = self.get_machine_id(machine_pid) + + sample_pid = getattr(event, 'sample_pid', None) + if sample_pid is None: + sample_pid = -1 + elif sample_pid > 0x7fffffff: + sample_pid -= 0x100000000 + sample_tid = getattr(event, 'sample_tid', None) + if sample_tid is None: + sample_tid = -1 + elif sample_tid > 0x7fffffff: + sample_tid -= 0x100000000 + next_prev_pid = getattr(event, 'next_prev_pid', None) + if next_prev_pid is None: + next_prev_pid = -1 + elif next_prev_pid > 0x7fffffff: + next_prev_pid -= 0x100000000 + next_prev_tid = getattr(event, 'next_prev_tid', None) + if next_prev_tid is None: + next_prev_tid = -1 + elif next_prev_tid > 0x7fffffff: + next_prev_tid -= 0x100000000 + + th_a_id = self.get_thread_id(machine_id, sample_pid, sample_tid) + comm_a = "Unknown_comm" + if self.session: + try: + proc = self.session.find_thread(sample_pid, sample_tid) + if proc: + comm_a_name = proc.comm() + if comm_a_name: + comm_a = comm_a_name + except TypeError: + pass + comm_a_id = self.get_comm_id(comm_a, th_a_id) + + th_b_id = self.get_thread_id(machine_id, next_prev_pid, next_prev_tid) + comm_b = "Unknown_comm" + if self.session: + try: + proc = self.session.find_thread(next_prev_pid, next_prev_tid) + if proc: + comm_b_name = proc.comm() + if comm_b_name: + comm_b = comm_b_name + except TypeError: + pass + comm_b_id = self.get_comm_id(comm_b, th_b_id) + + if out: + th_out_id = th_a_id + comm_out_id = comm_a_id + th_in_id = th_b_id + comm_in_id = comm_b_id + else: + th_out_id = th_b_id + comm_out_id = comm_b_id + th_in_id = th_a_id + comm_in_id = comm_a_id + + self._exec( + "INSERT INTO context_switches VALUES (NULL, ?, ?, ?, ?, ?, ?, ?, ?)", + (machine_id, getattr(event, 'sample_time', 0), getattr(event, 'sample_cpu', 0) or 0, + th_out_id, comm_out_id, th_in_id, comm_in_id, flags) + ) + + def finish(self) -> None: + """Create indexes and update summary metadata after processing events. + + pcpid_idx, pid_idx, and comms.has_calls are required by + exported-sql-viewer.py to filter the command dropdown to threads with + call graphs and to expand parent/child call trees efficiently. + """ + self.con.execute("CREATE INDEX IF NOT EXISTS pcpid_idx ON calls (parent_call_path_id)") + self.con.execute("CREATE INDEX IF NOT EXISTS pid_idx ON calls (parent_id)") + self.con.execute("ALTER TABLE comms ADD has_calls boolean") + self.con.execute("UPDATE comms SET has_calls = 1 WHERE comms.id IN " + "(SELECT DISTINCT comm_id FROM calls)") + + def commit(self) -> None: + """Commit transaction.""" + self.con.commit() + + def close(self) -> None: + """Close connection.""" + self.con.close() + + +if __name__ == "__main__": + ap = argparse.ArgumentParser( + description="Export perf data to a sqlite3 database") + ap.add_argument("-i", "--input", default="perf.data", + help="Input file name") + ap.add_argument("-o", "--output", default="perf.db", + help="Output database name") + ap.add_argument("--itrace", help="Instruction Tracing options (e.g. crpt)") + args = ap.parse_args() + + try: + fd = os.open(args.output, os.O_CREAT | os.O_EXCL | os.O_WRONLY, 0o600) + except FileExistsError: + print(f"Error: {args.output} already exists") + sys.exit(1) + + tmp_dir = None + exporter = None + succeeded = False + try: + tmp_dir = tempfile.mkdtemp(prefix="perf-export-sqlite-") + tmp_db = os.path.join(tmp_dir, "perf.db") + exporter = DatabaseExporter(tmp_db) + session = perf.session(perf.data(args.input), + sample=exporter.process_event, + context_switch=exporter.process_context_switch, + call_return=exporter.process_call_return, + itrace=args.itrace) + exporter.session = session + try: + session.process_events() + finally: + # Break the C perf.session -> bound method -> exporter reference cycle. + exporter.session = None + exporter.finish() + exporter.commit() + exporter.close() + exporter = None + with open(tmp_db, "rb") as src, os.fdopen(fd, "wb", closefd=False) as dst: + shutil.copyfileobj(src, dst) + succeeded = True + print(f"Successfully exported to {args.output}") + except (OSError, RuntimeError, ValueError, sqlite3.Error): + import traceback + traceback.print_exc() + finally: + if exporter is not None: + exporter.session = None + try: + exporter.close() + except (OSError, sqlite3.Error): + pass + try: + os.close(fd) + except OSError: + pass + if tmp_dir is not None: + shutil.rmtree(tmp_dir, ignore_errors=True) + if not succeeded: + try: + os.remove(args.output) + except OSError: + pass + if not succeeded: + sys.exit(1) diff --git a/tools/perf/python/exported-sql-viewer.py b/tools/perf/python/exported-sql-viewer.py new file mode 100755 index 000000000000..f31aa45d9a54 --- /dev/null +++ b/tools/perf/python/exported-sql-viewer.py @@ -0,0 +1,5103 @@ +#!/usr/bin/env python +# SPDX-License-Identifier: GPL-2.0 +"""exported-sql-viewer.py: view data from sql database.""" +# Copyright (c) 2014-2018, Intel Corporation. + +# To use this script you will need to have exported data using either the +# export-to-sqlite.py or the export-to-postgresql.py script. Refer to those +# scripts for details. +# +# Following on from the example in the export scripts, a +# call-graph can be displayed for the pt_example database like this: +# +# python exported-sql-viewer.py pt_example +# +# Note that for PostgreSQL, this script supports connecting to remote databases +# by setting hostname, port, username, password, and dbname e.g. +# +# python exported-sql-viewer.py \ +# "hostname=myhost username=myuser password=mypassword dbname=pt_example" +# +# The result is a GUI window with a tree representing a context-sensitive +# call-graph. Expanding a couple of levels of the tree and adjusting column +# widths to suit will display something like: +# +# Call Graph: pt_example +# Call Path Object Count Time(ns) Time(%) Branch Count Branch Count(%) +# v- ls +# v- 2638:2638 +# v- _start ld-2.19.so 1 10074071 100.0 211135 100.0 +# |- unknown unknown 1 13198 0.1 1 0.0 +# >- _dl_start ld-2.19.so 1 1400980 13.9 19637 9.3 +# >- _d_linit_internal ld-2.19.so 1 448152 4.4 11094 5.3 +# v-__libc_start_main@plt ls 1 8211741 81.5 180397 85.4 +# >- _dl_fixup ld-2.19.so 1 7607 0.1 108 0.1 +# >- __cxa_atexit libc-2.19.so 1 11737 0.1 10 0.0 +# >- __libc_csu_init ls 1 10354 0.1 10 0.0 +# |- _setjmp libc-2.19.so 1 0 0.0 4 0.0 +# v- main ls 1 8182043 99.6 180254 99.9 +# +# Points to note: +# The top level is a command name (comm) +# The next level is a thread (pid:tid) +# Subsequent levels are functions +# 'Count' is the number of calls +# 'Time' is the elapsed time until the function returns +# Percentages are relative to the level above +# 'Branch Count' is the total number of branches for that function and all +# functions that it calls + +# There is also a "All branches" report, which displays branches and +# possibly disassembly. However, presently, the only supported disassembler is +# Intel XED, and additionally the object code must be present in perf build ID +# cache. To use Intel XED, libxed.so must be present. To build and install +# libxed.so: +# git clone https://github.com/intelxed/mbuild.git mbuild +# git clone https://github.com/intelxed/xed +# cd xed +# ./mfile.py --share +# sudo ./mfile.py --prefix=/usr/local install +# sudo ldconfig +# +# Example report: +# +# Time CPU Command PID TID Branch Type In Tx Branch +# 8107675239590 2 ls 22011 22011 return from interrupt No ffffffff86a00a67 native_irq_return_iret ([kernel]) -> 7fab593ea260 _start (ld-2.19.so) +# 7fab593ea260 48 89 e7 mov %rsp, %rdi +# 8107675239899 2 ls 22011 22011 hardware interrupt No 7fab593ea260 _start (ld-2.19.so) -> ffffffff86a012e0 page_fault ([kernel]) +# 8107675241900 2 ls 22011 22011 return from interrupt No ffffffff86a00a67 native_irq_return_iret ([kernel]) -> 7fab593ea260 _start (ld-2.19.so) +# 7fab593ea260 48 89 e7 mov %rsp, %rdi +# 7fab593ea263 e8 c8 06 00 00 callq 0x7fab593ea930 +# 8107675241900 2 ls 22011 22011 call No 7fab593ea263 _start+0x3 (ld-2.19.so) -> 7fab593ea930 _dl_start (ld-2.19.so) +# 7fab593ea930 55 pushq %rbp +# 7fab593ea931 48 89 e5 mov %rsp, %rbp +# 7fab593ea934 41 57 pushq %r15 +# 7fab593ea936 41 56 pushq %r14 +# 7fab593ea938 41 55 pushq %r13 +# 7fab593ea93a 41 54 pushq %r12 +# 7fab593ea93c 53 pushq %rbx +# 7fab593ea93d 48 89 fb mov %rdi, %rbx +# 7fab593ea940 48 83 ec 68 sub $0x68, %rsp +# 7fab593ea944 0f 31 rdtsc +# 7fab593ea946 48 c1 e2 20 shl $0x20, %rdx +# 7fab593ea94a 89 c0 mov %eax, %eax +# 7fab593ea94c 48 09 c2 or %rax, %rdx +# 7fab593ea94f 48 8b 05 1a 15 22 00 movq 0x22151a(%rip), %rax +# 8107675242232 2 ls 22011 22011 hardware interrupt No 7fab593ea94f _dl_start+0x1f (ld-2.19.so) -> ffffffff86a012e0 page_fault ([kernel]) +# 8107675242900 2 ls 22011 22011 return from interrupt No ffffffff86a00a67 native_irq_return_iret ([kernel]) -> 7fab593ea94f _dl_start+0x1f (ld-2.19.so) +# 7fab593ea94f 48 8b 05 1a 15 22 00 movq 0x22151a(%rip), %rax +# 7fab593ea956 48 89 15 3b 13 22 00 movq %rdx, 0x22133b(%rip) +# 8107675243232 2 ls 22011 22011 hardware interrupt No 7fab593ea956 _dl_start+0x26 (ld-2.19.so) -> ffffffff86a012e0 page_fault ([kernel]) + +from __future__ import print_function + +import sys +# Only change warnings if the python -W option was not used +if not sys.warnoptions: + import warnings + # PySide2 causes deprecation warnings, ignore them. + warnings.filterwarnings("ignore", category=DeprecationWarning) +import argparse +import weakref +import threading +import pickle +glb_nsz = 16 +import re +import os +import random +import copy +import math +from libxed import LibXED + +import importlib +from typing import Any + +class _QtBase: + def __init__(self, *args: Any, **kwargs: Any) -> None: + pass + def __call__(self, *args: Any, **kwargs: Any) -> Any: + return self + def __getattr__(self, name: str) -> Any: + return _QtBase() + def mousePressEvent(self, event: Any) -> None: + pass + def mouseMoveEvent(self, event: Any) -> None: + pass + def mouseReleaseEvent(self, event: Any) -> None: + pass + def resizeEvent(self, event: Any) -> None: + pass + def changeEvent(self, event: Any) -> None: + pass + +class _QtWidget(_QtBase): + pass + +class _QtMdiSubWindow(_QtWidget): + pass + +QAbstractItemModel: Any = _QtBase +QAbstractItemView: Any = _QtBase +QAbstractTableModel: Any = _QtBase +QAction: Any = _QtBase +QApplication: Any = _QtBase +QCheckBox: Any = _QtBase +QColor: Any = _QtBase +QComboBox: Any = _QtBase +QDialog: Any = _QtWidget +QEvent: Any = _QtBase +QFont: Any = _QtBase +QFontMetrics: Any = _QtBase +QGraphicsItem: Any = _QtBase +QGraphicsLineItem: Any = _QtBase +QGraphicsScene: Any = _QtBase +QGraphicsSimpleTextItem: Any = _QtBase +QGraphicsView: Any = _QtBase +QGridLayout: Any = _QtBase +QHBoxLayout: Any = _QtBase +QKeySequence: Any = _QtBase +QLabel: Any = _QtBase +QLineEdit: Any = _QtBase +QMainWindow: Any = _QtWidget +QMdiArea: Any = _QtBase +QMdiSubWindow: Any = _QtMdiSubWindow +QMenu: Any = _QtBase +QMessageBox: Any = _QtBase +QModelIndex: Any = _QtBase +QObject: Any = _QtBase +QPalette: Any = _QtBase +QPoint: Any = _QtBase +QPointF: Any = _QtBase +QProgressBar: Any = _QtBase +QPushButton: Any = _QtBase +QRect: Any = _QtBase +QRectF: Any = _QtBase +QRubberBand: Any = _QtBase +QSize: Any = _QtBase +QSizePolicy: Any = _QtBase +QSortFilterProxyModel: Any = _QtBase +QSpinBox: Any = _QtBase +QSplitter: Any = _QtWidget +QSqlDatabase: Any = _QtBase +QSqlQuery: Any = _QtBase +QStyle: Any = _QtBase +Qt: Any = _QtBase +QTableView: Any = _QtWidget +QTextBrowser: Any = _QtWidget +QThread: Any = _QtBase +QToolButton: Any = _QtBase +QTreeView: Any = _QtWidget +QVBoxLayout: Any = _QtBase +qVersion: Any = _QtBase +QWidget: Any = _QtWidget +Signal: Any = _QtBase + +pyside_version_1 = False +for _pkg in ("PySide6", "PySide2"): + try: + for _mod_name in ("QtCore", "QtGui", "QtSql", "QtWidgets"): + _mod = importlib.import_module(f"{_pkg}.{_mod_name}") + for _sym in dir(_mod): + if not _sym.startswith("_"): + globals()[_sym] = getattr(_mod, _sym) + break + except ImportError: + pass + + +from decimal import Decimal, ROUND_HALF_UP +from ctypes import create_string_buffer, addressof, sizeof, \ + c_void_p, c_bool, c_char, c_longlong +from multiprocessing import Process, Array, Value, Event + + +def printerr(*args, **keyword_args): + print(*args, file=sys.stderr, **keyword_args) + +# Data formatting helpers + +def tohex(ip): + if ip < 0: + ip += 1 << 64 + return "%x" % ip + +def offstr(offset): + if offset: + return "+0x%x" % offset + return "" + +def dsoname(name): + if name == "[kernel.kallsyms]": + return "[kernel]" + return name + +def findnth(s, sub, n, offs=0): + pos = s.find(sub) + if pos < 0: + return pos + if n <= 1: + return offs + pos + return findnth(s[pos + 1:], sub, n - 1, offs + pos + 1) + +# Percent to one decimal place + +def PercentToOneDP(n, d): + if not d: + return "0.0" + x = (n * Decimal(100)) / d + return str(x.quantize(Decimal(".1"), rounding=ROUND_HALF_UP)) + +# Helper for queries that must not fail + +def QueryExec(query, stmt): + ret = query.exec_(stmt) + if not ret: + raise RuntimeError("Query failed: " + query.lastError().text()) + +# Background thread + +class Thread(QThread): + + done = Signal(object) + + def __init__(self, task, param=None, parent=None): + super(Thread, self).__init__(parent) + self.task = task + self.param = param + + def run(self): + while True: + if self.param is None: + done, result = self.task() + else: + done, result = self.task(self.param) + self.done.emit(result) + if done: + break + +# Tree data model + +class TreeModel(QAbstractItemModel): + + def __init__(self, glb, params, parent=None): + super(TreeModel, self).__init__(parent) + self.glb = glb + self.params = params + self.root = self.GetRoot() + self.last_row_read = 0 + + def Item(self, parent): + if parent.isValid(): + return parent.internalPointer() + else: + return self.root + + def rowCount(self, parent): + result = self.Item(parent).childCount() + if result < 0: + result = 0 + self.dataChanged.emit(parent, parent) + return result + + def hasChildren(self, parent): + return self.Item(parent).hasChildren() + + def headerData(self, section, orientation, role): + if role == Qt.TextAlignmentRole: + return self.columnAlignment(section) + if role != Qt.DisplayRole: + return None + if orientation != Qt.Horizontal: + return None + return self.columnHeader(section) + + def parent(self, child): + child_item = child.internalPointer() + if child_item is self.root: + return QModelIndex() + parent_item = child_item.getParentItem() + return self.createIndex(parent_item.getRow(), 0, parent_item) + + def index(self, row, column, parent): + child_item = self.Item(parent).getChildItem(row) + return self.createIndex(row, column, child_item) + + def DisplayData(self, item, index): + return item.getData(index.column()) + + def FetchIfNeeded(self, row): + if row > self.last_row_read: + self.last_row_read = row + if row + 10 >= self.root.child_count: + self.fetcher.Fetch(glb_chunk_sz) + + def columnAlignment(self, _column): + return Qt.AlignLeft + + def columnFont(self, _column): + return None + + def data(self, index, role): + if role == Qt.TextAlignmentRole: + return self.columnAlignment(index.column()) + if role == Qt.FontRole: + return self.columnFont(index.column()) + if role != Qt.DisplayRole: + return None + item = index.internalPointer() + return self.DisplayData(item, index) + +# Table data model + +class TableModel(QAbstractTableModel): + + def __init__(self, parent=None): + super(TableModel, self).__init__(parent) + self.child_count = 0 + self.child_items = [] + self.last_row_read = 0 + + def Item(self, parent): + if parent.isValid(): + return parent.internalPointer() + else: + return self + + def rowCount(self, _parent): + return self.child_count + + def headerData(self, section, orientation, role): + if role == Qt.TextAlignmentRole: + return self.columnAlignment(section) + if role != Qt.DisplayRole: + return None + if orientation != Qt.Horizontal: + return None + return self.columnHeader(section) + + def index(self, row, column, _parent): + return self.createIndex(row, column, self.child_items[row]) + + def DisplayData(self, item, index): + return item.getData(index.column()) + + def FetchIfNeeded(self, row): + if row > self.last_row_read: + self.last_row_read = row + if row + 10 >= self.child_count: + self.fetcher.Fetch(glb_chunk_sz) + + def columnAlignment(self, _column): + return Qt.AlignLeft + + def columnFont(self, _column): + return None + + def data(self, index, role): + if role == Qt.TextAlignmentRole: + return self.columnAlignment(index.column()) + if role == Qt.FontRole: + return self.columnFont(index.column()) + if role != Qt.DisplayRole: + return None + item = index.internalPointer() + return self.DisplayData(item, index) + +# Model cache + +model_cache: Any = weakref.WeakValueDictionary() +model_cache_lock = threading.Lock() + +def LookupCreateModel(model_name, create_fn): + model_cache_lock.acquire() + try: + model = model_cache[model_name] + except KeyError: + model = None + if model is None: + model = create_fn() + model_cache[model_name] = model + model_cache_lock.release() + return model + +def LookupModel(model_name): + model_cache_lock.acquire() + try: + model = model_cache[model_name] + except KeyError: + model = None + model_cache_lock.release() + return model + +# Find bar + +class FindBar(): + + def __init__(self, parent, finder, is_reg_expr=False): + self.finder = finder + self.context = [] + self.last_value = None + self.last_pattern = None + + label = QLabel("Find:") + label.setSizePolicy(QSizePolicy.Fixed, QSizePolicy.Fixed) + + self.textbox = QComboBox() + self.textbox.setEditable(True) + self.textbox.currentIndexChanged.connect(self.ValueChanged) + + self.progress = QProgressBar() + self.progress.setRange(0, 0) + self.progress.hide() + + if is_reg_expr: + self.pattern = QCheckBox("Regular Expression") + else: + self.pattern = QCheckBox("Pattern") + self.pattern.setSizePolicy(QSizePolicy.Fixed, QSizePolicy.Fixed) + + self.next_button = QToolButton() + self.next_button.setIcon(parent.style().standardIcon(QStyle.SP_ArrowDown)) + self.next_button.released.connect(lambda: self.NextPrev(1)) + + self.prev_button = QToolButton() + self.prev_button.setIcon(parent.style().standardIcon(QStyle.SP_ArrowUp)) + self.prev_button.released.connect(lambda: self.NextPrev(-1)) + + self.close_button = QToolButton() + self.close_button.setIcon(parent.style().standardIcon(QStyle.SP_DockWidgetCloseButton)) + self.close_button.released.connect(self.Deactivate) + + self.hbox = QHBoxLayout() + self.hbox.setContentsMargins(0, 0, 0, 0) + + self.hbox.addWidget(label) + self.hbox.addWidget(self.textbox) + self.hbox.addWidget(self.progress) + self.hbox.addWidget(self.pattern) + self.hbox.addWidget(self.next_button) + self.hbox.addWidget(self.prev_button) + self.hbox.addWidget(self.close_button) + + self.bar = QWidget() + self.bar.setLayout(self.hbox) + self.bar.hide() + + def Widget(self): + return self.bar + + def Activate(self): + self.bar.show() + self.textbox.lineEdit().selectAll() + self.textbox.setFocus() + + def Deactivate(self): + self.bar.hide() + + def Busy(self): + self.textbox.setEnabled(False) + self.pattern.hide() + self.next_button.hide() + self.prev_button.hide() + self.progress.show() + + def Idle(self): + self.textbox.setEnabled(True) + self.progress.hide() + self.pattern.show() + self.next_button.show() + self.prev_button.show() + + def Find(self, direction): + value = self.textbox.currentText() + pattern = self.pattern.isChecked() + self.last_value = value + self.last_pattern = pattern + self.finder.Find(value, direction, pattern, self.context) + + def ValueChanged(self): + _value = self.textbox.currentText() + pattern = self.pattern.isChecked() + index = self.textbox.currentIndex() + data = self.textbox.itemData(index) + # Store the pattern in the combo box to keep it with the text value + if data == None: + self.textbox.setItemData(index, pattern) + else: + self.pattern.setChecked(data) + self.Find(0) + + def NextPrev(self, direction): + value = self.textbox.currentText() + pattern = self.pattern.isChecked() + if value != self.last_value: + index = self.textbox.findText(value) + # Allow for a button press before the value has been added to the combo box + if index < 0: + index = self.textbox.count() + self.textbox.addItem(value, pattern) + self.textbox.setCurrentIndex(index) + return + else: + self.textbox.setItemData(index, pattern) + elif pattern != self.last_pattern: + # Keep the pattern recorded in the combo box up to date + index = self.textbox.currentIndex() + self.textbox.setItemData(index, pattern) + self.Find(direction) + + def NotFound(self): + QMessageBox.information(self.bar, "Find", "'" + self.textbox.currentText() + "' not found") + +# Context-sensitive call graph data model item base + +class CallGraphLevelItemBase(object): + def Select(self) -> None: + pass + def data(self, _column: int, _role: int) -> Any: + return None + + def __init__(self, glb, params, row, parent_item): + self.glb = glb + self.params = params + self.row = row + self.parent_item = parent_item + self.query_done = False + self.child_count = 0 + self.child_items = [] + if parent_item: + self.level = parent_item.level + 1 + else: + self.level = 0 + + def getChildItem(self, row): + return self.child_items[row] + + def getParentItem(self): + return self.parent_item + + def getRow(self): + return self.row + + def childCount(self): + if not self.query_done: + self.Select() + if not self.child_count: + return -1 + return self.child_count + + def hasChildren(self): + if not self.query_done: + return True + return self.child_count > 0 + + def getData(self, column): + return self.data[column] + +# Context-sensitive call graph data model level 2+ item base + +class CallGraphLevelTwoPlusItemBase(CallGraphLevelItemBase): + + def __init__(self, glb, params, row, comm_id, thread_id, call_path_id, time, insn_cnt, cyc_cnt, branch_count, parent_item): + super(CallGraphLevelTwoPlusItemBase, self).__init__(glb, params, row, parent_item) + self.comm_id = comm_id + self.thread_id = thread_id + self.call_path_id = call_path_id + self.insn_cnt = insn_cnt + self.cyc_cnt = cyc_cnt + self.branch_count = branch_count + self.time = time + + def Select(self): + self.query_done = True + query = QSqlQuery(self.glb.db) + if self.params.have_ipc: + ipc_str = ", SUM(insn_count), SUM(cyc_count)" + else: + ipc_str = "" + QueryExec(query, "SELECT call_path_id, name, short_name, COUNT(calls.id), SUM(return_time - call_time)" + ipc_str + ", SUM(branch_count)" + " FROM calls" + " INNER JOIN call_paths ON calls.call_path_id = call_paths.id" + " INNER JOIN symbols ON call_paths.symbol_id = symbols.id" + " INNER JOIN dsos ON symbols.dso_id = dsos.id" + " WHERE parent_call_path_id = " + str(self.call_path_id) + + " AND comm_id = " + str(self.comm_id) + + " AND thread_id = " + str(self.thread_id) + + " GROUP BY call_path_id, name, short_name" + " ORDER BY call_path_id") + while query.next(): + if self.params.have_ipc: + insn_cnt = int(query.value(5)) + cyc_cnt = int(query.value(6)) + branch_count = int(query.value(7)) + else: + insn_cnt = 0 + cyc_cnt = 0 + branch_count = int(query.value(5)) + child_item = CallGraphLevelThreeItem(self.glb, self.params, self.child_count, self.comm_id, self.thread_id, query.value(0), query.value(1), query.value(2), query.value(3), int(query.value(4)), insn_cnt, cyc_cnt, branch_count, self) + self.child_items.append(child_item) + self.child_count += 1 + +# Context-sensitive call graph data model level three item + +class CallGraphLevelThreeItem(CallGraphLevelTwoPlusItemBase): + + def __init__(self, glb, params, row, comm_id, thread_id, call_path_id, name, dso, count, time, insn_cnt, cyc_cnt, branch_count, parent_item): + super(CallGraphLevelThreeItem, self).__init__(glb, params, row, comm_id, thread_id, call_path_id, time, insn_cnt, cyc_cnt, branch_count, parent_item) + dso = dsoname(dso) + if self.params.have_ipc: + insn_pcnt = PercentToOneDP(insn_cnt, parent_item.insn_cnt) + cyc_pcnt = PercentToOneDP(cyc_cnt, parent_item.cyc_cnt) + br_pcnt = PercentToOneDP(branch_count, parent_item.branch_count) + ipc = CalcIPC(cyc_cnt, insn_cnt) + self.data = [ name, dso, str(count), str(time), PercentToOneDP(time, parent_item.time), str(insn_cnt), insn_pcnt, str(cyc_cnt), cyc_pcnt, ipc, str(branch_count), br_pcnt ] + else: + self.data = [ name, dso, str(count), str(time), PercentToOneDP(time, parent_item.time), str(branch_count), PercentToOneDP(branch_count, parent_item.branch_count) ] + self.dbid = call_path_id + +# Context-sensitive call graph data model level two item + +class CallGraphLevelTwoItem(CallGraphLevelTwoPlusItemBase): + + def __init__(self, glb, params, row, comm_id, thread_id, pid, tid, parent_item): + super(CallGraphLevelTwoItem, self).__init__(glb, params, row, comm_id, thread_id, 1, 0, 0, 0, 0, parent_item) + if self.params.have_ipc: + self.data = [str(pid) + ":" + str(tid), "", "", "", "", "", "", "", "", "", "", ""] + else: + self.data = [str(pid) + ":" + str(tid), "", "", "", "", "", ""] + self.dbid = thread_id + + def Select(self): + super(CallGraphLevelTwoItem, self).Select() + for child_item in self.child_items: + self.time += child_item.time + self.insn_cnt += child_item.insn_cnt + self.cyc_cnt += child_item.cyc_cnt + self.branch_count += child_item.branch_count + for child_item in self.child_items: + child_item.data[4] = PercentToOneDP(child_item.time, self.time) + if self.params.have_ipc: + child_item.data[6] = PercentToOneDP(child_item.insn_cnt, self.insn_cnt) + child_item.data[8] = PercentToOneDP(child_item.cyc_cnt, self.cyc_cnt) + child_item.data[11] = PercentToOneDP(child_item.branch_count, self.branch_count) + else: + child_item.data[6] = PercentToOneDP(child_item.branch_count, self.branch_count) + +# Context-sensitive call graph data model level one item + +class CallGraphLevelOneItem(CallGraphLevelItemBase): + + def __init__(self, glb, params, row, comm_id, comm, parent_item): + super(CallGraphLevelOneItem, self).__init__(glb, params, row, parent_item) + if self.params.have_ipc: + self.data = [comm, "", "", "", "", "", "", "", "", "", "", ""] + else: + self.data = [comm, "", "", "", "", "", ""] + self.dbid = comm_id + + def Select(self): + self.query_done = True + query = QSqlQuery(self.glb.db) + QueryExec(query, "SELECT thread_id, pid, tid" + " FROM comm_threads" + " INNER JOIN threads ON thread_id = threads.id" + " WHERE comm_id = " + str(self.dbid)) + while query.next(): + child_item = CallGraphLevelTwoItem(self.glb, self.params, self.child_count, self.dbid, query.value(0), query.value(1), query.value(2), self) + self.child_items.append(child_item) + self.child_count += 1 + +# Context-sensitive call graph data model root item + +class CallGraphRootItem(CallGraphLevelItemBase): + + def __init__(self, glb, params): + super(CallGraphRootItem, self).__init__(glb, params, 0, None) + self.dbid = 0 + self.query_done = True + if_has_calls = "" + if IsSelectable(glb.db, "comms", columns = "has_calls"): + if_has_calls = " WHERE has_calls = " + glb.dbref.TRUE + query = QSqlQuery(glb.db) + QueryExec(query, "SELECT id, comm FROM comms" + if_has_calls) + while query.next(): + if not query.value(0): + continue + child_item = CallGraphLevelOneItem(glb, params, self.child_count, query.value(0), query.value(1), self) + self.child_items.append(child_item) + self.child_count += 1 + +# Call graph model parameters + +class CallGraphModelParams(): + + def __init__(self, glb, _parent=None): + self.have_ipc = IsSelectable(glb.db, "calls", columns = "insn_count, cyc_count") + +# Context-sensitive call graph data model base + +class CallGraphModelBase(TreeModel): + + def __init__(self, glb, parent=None): + super(CallGraphModelBase, self).__init__(glb, CallGraphModelParams(glb), parent) + + def FindSelect(self, value, pattern, query): + if pattern: + # postgresql and sqlite pattern patching differences: + # postgresql LIKE is case sensitive but sqlite LIKE is not + # postgresql LIKE allows % and _ to be escaped with \ but sqlite LIKE does not + # postgresql supports ILIKE which is case insensitive + # sqlite supports GLOB (text only) which uses * and ? and is case sensitive + if not self.glb.dbref.is_sqlite3: + # Escape % and _ + s = value.replace("%", "\\%") + s = s.replace("_", "\\_") + # Translate * and ? into SQL LIKE pattern characters % and _ + if sys.version_info[0] == 3: + trans = str.maketrans("*?", "%_") + else: + trans = bytes.maketrans("*?", "%_") + match = " LIKE '" + str(s).translate(trans) + "'" + else: + match = " GLOB '" + str(value) + "'" + else: + match = " = '" + str(value) + "'" + self.DoFindSelect(query, match) + + def Found(self, query, found): + if found: + return self.FindPath(query) + return [] + + def FindValue(self, value, pattern, query, last_value, last_pattern): + if last_value == value and pattern == last_pattern: + found = query.first() + else: + self.FindSelect(value, pattern, query) + found = query.next() + return self.Found(query, found) + + def FindNext(self, query): + found = query.next() + if not found: + found = query.first() + return self.Found(query, found) + + def FindPrev(self, query): + found = query.previous() + if not found: + found = query.last() + return self.Found(query, found) + + def FindThread(self, c): + if c.direction == 0 or c.value != c.last_value or c.pattern != c.last_pattern: + ids = self.FindValue(c.value, c.pattern, c.query, c.last_value, c.last_pattern) + elif c.direction > 0: + ids = self.FindNext(c.query) + else: + ids = self.FindPrev(c.query) + return (True, ids) + + def Find(self, value, direction, pattern, context, callback): + class Context(): + def __init__(self, *x): + self.value, self.direction, self.pattern, self.query, self.last_value, self.last_pattern = x + def Update(self, *x): + self.value, self.direction, self.pattern, self.last_value, self.last_pattern = x + (self.value, self.pattern) + if len(context): + context[0].Update(value, direction, pattern) + else: + context.append(Context(value, direction, pattern, QSqlQuery(self.glb.db), None, None)) + # Use a thread so the UI is not blocked during the SELECT + thread = Thread(self.FindThread, context[0]) + thread.done.connect(lambda ids, t=thread, c=callback: self.FindDone(t, c, ids), Qt.QueuedConnection) + thread.start() + + def FindDone(self, _thread, callback, ids): + callback(ids) + +# Context-sensitive call graph data model + +class CallGraphModel(CallGraphModelBase): + + def GetRoot(self): + return CallGraphRootItem(self.glb, self.params) + + def columnCount(self, _parent=None): + if self.params.have_ipc: + return 12 + else: + return 7 + + def columnHeader(self, column): + if self.params.have_ipc: + headers = ["Call Path", "Object", "Count ", "Time (ns) ", "Time (%) ", "Insn Cnt", "Insn Cnt (%)", "Cyc Cnt", "Cyc Cnt (%)", "IPC", "Branch Count ", "Branch Count (%) "] + else: + headers = ["Call Path", "Object", "Count ", "Time (ns) ", "Time (%) ", "Branch Count ", "Branch Count (%) "] + return headers[column] + + def columnAlignment(self, column): + if self.params.have_ipc: + alignment = [ Qt.AlignLeft, Qt.AlignLeft, Qt.AlignRight, Qt.AlignRight, Qt.AlignRight, Qt.AlignRight, Qt.AlignRight, Qt.AlignRight, Qt.AlignRight, Qt.AlignRight, Qt.AlignRight, Qt.AlignRight ] + else: + alignment = [ Qt.AlignLeft, Qt.AlignLeft, Qt.AlignRight, Qt.AlignRight, Qt.AlignRight, Qt.AlignRight, Qt.AlignRight ] + return alignment[column] + + def DoFindSelect(self, query, match): + QueryExec(query, "SELECT call_path_id, comm_id, thread_id" + " FROM calls" + " INNER JOIN call_paths ON calls.call_path_id = call_paths.id" + " INNER JOIN symbols ON call_paths.symbol_id = symbols.id" + " WHERE calls.id <> 0" + " AND symbols.name" + match + + " GROUP BY comm_id, thread_id, call_path_id" + " ORDER BY comm_id, thread_id, call_path_id") + + def FindPath(self, query): + # Turn the query result into a list of ids that the tree view can walk + # to open the tree at the right place. + ids = [] + parent_id = query.value(0) + while parent_id: + ids.insert(0, parent_id) + q2 = QSqlQuery(self.glb.db) + QueryExec(q2, "SELECT parent_id" + " FROM call_paths" + " WHERE id = " + str(parent_id)) + if not q2.next(): + break + parent_id = q2.value(0) + # The call path root is not used + if ids[0] == 1: + del ids[0] + ids.insert(0, query.value(2)) + ids.insert(0, query.value(1)) + return ids + +# Call tree data model level 2+ item base + +class CallTreeLevelTwoPlusItemBase(CallGraphLevelItemBase): + + def __init__(self, glb, params, row, comm_id, thread_id, calls_id, call_time, time, insn_cnt, cyc_cnt, branch_count, parent_item): + super(CallTreeLevelTwoPlusItemBase, self).__init__(glb, params, row, parent_item) + self.comm_id = comm_id + self.thread_id = thread_id + self.calls_id = calls_id + self.call_time = call_time + self.time = time + self.insn_cnt = insn_cnt + self.cyc_cnt = cyc_cnt + self.branch_count = branch_count + + def Select(self): + self.query_done = True + if self.calls_id == 0: + comm_thread = " AND comm_id = " + str(self.comm_id) + " AND thread_id = " + str(self.thread_id) + else: + comm_thread = "" + if self.params.have_ipc: + ipc_str = ", insn_count, cyc_count" + else: + ipc_str = "" + query = QSqlQuery(self.glb.db) + QueryExec(query, "SELECT calls.id, name, short_name, call_time, return_time - call_time" + ipc_str + ", branch_count" + " FROM calls" + " INNER JOIN call_paths ON calls.call_path_id = call_paths.id" + " INNER JOIN symbols ON call_paths.symbol_id = symbols.id" + " INNER JOIN dsos ON symbols.dso_id = dsos.id" + " WHERE calls.parent_id = " + str(self.calls_id) + comm_thread + + " ORDER BY call_time, calls.id") + while query.next(): + if self.params.have_ipc: + insn_cnt = int(query.value(5)) + cyc_cnt = int(query.value(6)) + branch_count = int(query.value(7)) + else: + insn_cnt = 0 + cyc_cnt = 0 + branch_count = int(query.value(5)) + child_item = CallTreeLevelThreeItem(self.glb, self.params, self.child_count, self.comm_id, self.thread_id, query.value(0), query.value(1), query.value(2), query.value(3), int(query.value(4)), insn_cnt, cyc_cnt, branch_count, self) + self.child_items.append(child_item) + self.child_count += 1 + +# Call tree data model level three item + +class CallTreeLevelThreeItem(CallTreeLevelTwoPlusItemBase): + + def __init__(self, glb, params, row, comm_id, thread_id, calls_id, name, dso, call_time, time, insn_cnt, cyc_cnt, branch_count, parent_item): + super(CallTreeLevelThreeItem, self).__init__(glb, params, row, comm_id, thread_id, calls_id, call_time, time, insn_cnt, cyc_cnt, branch_count, parent_item) + dso = dsoname(dso) + if self.params.have_ipc: + insn_pcnt = PercentToOneDP(insn_cnt, parent_item.insn_cnt) + cyc_pcnt = PercentToOneDP(cyc_cnt, parent_item.cyc_cnt) + br_pcnt = PercentToOneDP(branch_count, parent_item.branch_count) + ipc = CalcIPC(cyc_cnt, insn_cnt) + self.data = [ name, dso, str(call_time), str(time), PercentToOneDP(time, parent_item.time), str(insn_cnt), insn_pcnt, str(cyc_cnt), cyc_pcnt, ipc, str(branch_count), br_pcnt ] + else: + self.data = [ name, dso, str(call_time), str(time), PercentToOneDP(time, parent_item.time), str(branch_count), PercentToOneDP(branch_count, parent_item.branch_count) ] + self.dbid = calls_id + +# Call tree data model level two item + +class CallTreeLevelTwoItem(CallTreeLevelTwoPlusItemBase): + + def __init__(self, glb, params, row, comm_id, thread_id, pid, tid, parent_item): + super(CallTreeLevelTwoItem, self).__init__(glb, params, row, comm_id, thread_id, 0, 0, 0, 0, 0, 0, parent_item) + if self.params.have_ipc: + self.data = [str(pid) + ":" + str(tid), "", "", "", "", "", "", "", "", "", "", ""] + else: + self.data = [str(pid) + ":" + str(tid), "", "", "", "", "", ""] + self.dbid = thread_id + + def Select(self): + super(CallTreeLevelTwoItem, self).Select() + for child_item in self.child_items: + self.time += child_item.time + self.insn_cnt += child_item.insn_cnt + self.cyc_cnt += child_item.cyc_cnt + self.branch_count += child_item.branch_count + for child_item in self.child_items: + child_item.data[4] = PercentToOneDP(child_item.time, self.time) + if self.params.have_ipc: + child_item.data[6] = PercentToOneDP(child_item.insn_cnt, self.insn_cnt) + child_item.data[8] = PercentToOneDP(child_item.cyc_cnt, self.cyc_cnt) + child_item.data[11] = PercentToOneDP(child_item.branch_count, self.branch_count) + else: + child_item.data[6] = PercentToOneDP(child_item.branch_count, self.branch_count) + +# Call tree data model level one item + +class CallTreeLevelOneItem(CallGraphLevelItemBase): + + def __init__(self, glb, params, row, comm_id, comm, parent_item): + super(CallTreeLevelOneItem, self).__init__(glb, params, row, parent_item) + if self.params.have_ipc: + self.data = [comm, "", "", "", "", "", "", "", "", "", "", ""] + else: + self.data = [comm, "", "", "", "", "", ""] + self.dbid = comm_id + + def Select(self): + self.query_done = True + query = QSqlQuery(self.glb.db) + QueryExec(query, "SELECT thread_id, pid, tid" + " FROM comm_threads" + " INNER JOIN threads ON thread_id = threads.id" + " WHERE comm_id = " + str(self.dbid)) + while query.next(): + child_item = CallTreeLevelTwoItem(self.glb, self.params, self.child_count, self.dbid, query.value(0), query.value(1), query.value(2), self) + self.child_items.append(child_item) + self.child_count += 1 + +# Call tree data model root item + +class CallTreeRootItem(CallGraphLevelItemBase): + + def __init__(self, glb, params): + super(CallTreeRootItem, self).__init__(glb, params, 0, None) + self.dbid = 0 + self.query_done = True + if_has_calls = "" + if IsSelectable(glb.db, "comms", columns = "has_calls"): + if_has_calls = " WHERE has_calls = " + glb.dbref.TRUE + query = QSqlQuery(glb.db) + QueryExec(query, "SELECT id, comm FROM comms" + if_has_calls) + while query.next(): + if not query.value(0): + continue + child_item = CallTreeLevelOneItem(glb, params, self.child_count, query.value(0), query.value(1), self) + self.child_items.append(child_item) + self.child_count += 1 + +# Call Tree data model + +class CallTreeModel(CallGraphModelBase): + + def GetRoot(self): + return CallTreeRootItem(self.glb, self.params) + + def columnCount(self, _parent=None): + if self.params.have_ipc: + return 12 + else: + return 7 + + def columnHeader(self, column): + if self.params.have_ipc: + headers = ["Call Path", "Object", "Call Time", "Time (ns) ", "Time (%) ", "Insn Cnt", "Insn Cnt (%)", "Cyc Cnt", "Cyc Cnt (%)", "IPC", "Branch Count ", "Branch Count (%) "] + else: + headers = ["Call Path", "Object", "Call Time", "Time (ns) ", "Time (%) ", "Branch Count ", "Branch Count (%) "] + return headers[column] + + def columnAlignment(self, column): + if self.params.have_ipc: + alignment = [ Qt.AlignLeft, Qt.AlignLeft, Qt.AlignRight, Qt.AlignRight, Qt.AlignRight, Qt.AlignRight, Qt.AlignRight, Qt.AlignRight, Qt.AlignRight, Qt.AlignRight, Qt.AlignRight, Qt.AlignRight ] + else: + alignment = [ Qt.AlignLeft, Qt.AlignLeft, Qt.AlignRight, Qt.AlignRight, Qt.AlignRight, Qt.AlignRight, Qt.AlignRight ] + return alignment[column] + + def DoFindSelect(self, query, match): + QueryExec(query, "SELECT calls.id, comm_id, thread_id" + " FROM calls" + " INNER JOIN call_paths ON calls.call_path_id = call_paths.id" + " INNER JOIN symbols ON call_paths.symbol_id = symbols.id" + " WHERE calls.id <> 0" + " AND symbols.name" + match + + " ORDER BY comm_id, thread_id, call_time, calls.id") + + def FindPath(self, query): + # Turn the query result into a list of ids that the tree view can walk + # to open the tree at the right place. + ids = [] + parent_id = query.value(0) + while parent_id: + ids.insert(0, parent_id) + q2 = QSqlQuery(self.glb.db) + QueryExec(q2, "SELECT parent_id" + " FROM calls" + " WHERE id = " + str(parent_id)) + if not q2.next(): + break + parent_id = q2.value(0) + ids.insert(0, query.value(2)) + ids.insert(0, query.value(1)) + return ids + +# Vertical layout + +class HBoxLayout(QHBoxLayout): + + def __init__(self, *children): + super(HBoxLayout, self).__init__() + + self.layout().setContentsMargins(0, 0, 0, 0) + for child in children: + if child.isWidgetType(): + self.layout().addWidget(child) + else: + self.layout().addLayout(child) + +# Horizontal layout + +class VBoxLayout(QVBoxLayout): + + def __init__(self, *children): + super(VBoxLayout, self).__init__() + + self.layout().setContentsMargins(0, 0, 0, 0) + for child in children: + if child.isWidgetType(): + self.layout().addWidget(child) + else: + self.layout().addLayout(child) + +# Vertical layout widget + +class VBox(): + + def __init__(self, *children): + self.vbox = QWidget() + self.vbox.setLayout(VBoxLayout(*children)) + + def Widget(self): + return self.vbox + +# Tree window base + +class TreeWindowBase(QMdiSubWindow): + + def __init__(self, parent=None): + super(TreeWindowBase, self).__init__(parent) + + self.model = None + self.find_bar = None + + self.view = QTreeView() + self.view.setSelectionMode(QAbstractItemView.ContiguousSelection) + self.view.CopyCellsToClipboard = CopyTreeCellsToClipboard + + self.context_menu = TreeContextMenu(self.view) + + def DisplayFound(self, ids): + if not len(ids): + return False + parent = QModelIndex() + for dbid in ids: + found = False + n = self.model.rowCount(parent) + for row in range(n): + child = self.model.index(row, 0, parent) + if child.internalPointer().dbid == dbid: + found = True + self.view.setExpanded(parent, True) + self.view.setCurrentIndex(child) + parent = child + break + if not found: + break + return found + + def Find(self, value, direction, pattern, context): + self.view.setFocus() + self.find_bar.Busy() + self.model.Find(value, direction, pattern, context, self.FindDone) + + def FindDone(self, ids): + found = True + if not self.DisplayFound(ids): + found = False + self.find_bar.Idle() + if not found: + self.find_bar.NotFound() + + +# Context-sensitive call graph window + +class CallGraphWindow(TreeWindowBase): + + def __init__(self, glb, parent=None): + super(CallGraphWindow, self).__init__(parent) + + self.model = LookupCreateModel("Context-Sensitive Call Graph", lambda x=glb: CallGraphModel(x)) + + self.view.setModel(self.model) + + for c, w in ((0, 250), (1, 100), (2, 60), (3, 70), (4, 70), (5, 100)): + self.view.setColumnWidth(c, w) + + self.find_bar = FindBar(self, self) + + self.vbox = VBox(self.view, self.find_bar.Widget()) + + self.setWidget(self.vbox.Widget()) + + AddSubWindow(glb.mainwindow.mdi_area, self, "Context-Sensitive Call Graph") + +# Call tree window + +class CallTreeWindow(TreeWindowBase): + + def __init__(self, glb, parent=None, thread_at_time=None): + super(CallTreeWindow, self).__init__(parent) + + self.model = LookupCreateModel("Call Tree", lambda x=glb: CallTreeModel(x)) + + self.view.setModel(self.model) + + for c, w in ((0, 230), (1, 100), (2, 100), (3, 70), (4, 70), (5, 100)): + self.view.setColumnWidth(c, w) + + self.find_bar = FindBar(self, self) + + self.vbox = VBox(self.view, self.find_bar.Widget()) + + self.setWidget(self.vbox.Widget()) + + AddSubWindow(glb.mainwindow.mdi_area, self, "Call Tree") + + if thread_at_time: + self.DisplayThreadAtTime(*thread_at_time) + + def DisplayThreadAtTime(self, comm_id, thread_id, time): + parent = QModelIndex() + for dbid in (comm_id, thread_id): + found = False + n = self.model.rowCount(parent) + for row in range(n): + child = self.model.index(row, 0, parent) + if child.internalPointer().dbid == dbid: + found = True + self.view.setExpanded(parent, True) + self.view.setCurrentIndex(child) + parent = child + break + if not found: + return + found = False + while True: + n = self.model.rowCount(parent) + if not n: + return + last_child = None + for row in range(n): + self.view.setExpanded(parent, True) + child = self.model.index(row, 0, parent) + child_call_time = child.internalPointer().call_time + if child_call_time < time: + last_child = child + elif child_call_time == time: + self.view.setCurrentIndex(child) + return + elif child_call_time > time: + break + if not last_child: + if not found: + child = self.model.index(0, 0, parent) + self.view.setExpanded(parent, True) + self.view.setCurrentIndex(child) + return + found = True + self.view.setExpanded(parent, True) + self.view.setCurrentIndex(last_child) + parent = last_child + +# ExecComm() gets the comm_id of the command string that was set when the process exec'd i.e. the program name + +def ExecComm(db, thread_id, time): + query = QSqlQuery(db) + QueryExec(query, "SELECT comm_threads.comm_id, comms.c_time, comms.exec_flag" + " FROM comm_threads" + " INNER JOIN comms ON comms.id = comm_threads.comm_id" + " WHERE comm_threads.thread_id = " + str(thread_id) + + " ORDER BY comms.c_time, comms.id") + first = None + last = None + while query.next(): + if first is None: + first = query.value(0) + if query.value(2) and Decimal(query.value(1)) <= Decimal(time): + last = query.value(0) + if not(last is None): + return last + return first + +# Container for (x, y) data + +class XY(): + def __init__(self, x=0, y=0): + self.x = x + self.y = y + + def __str__(self): + return "XY({}, {})".format(str(self.x), str(self.y)) + +# Container for sub-range data + +class Subrange(): + def __init__(self, lo=0, hi=0): + self.lo = lo + self.hi = hi + + def __str__(self): + return "Subrange({}, {})".format(str(self.lo), str(self.hi)) + +# Graph data region base class + +class GraphDataRegion(object): + + def __init__(self, key, title = "", ordinal = ""): + self.key = key + self.title = title + self.ordinal = ordinal + +# Function to sort GraphDataRegion + +def GraphDataRegionOrdinal(data_region): + return data_region.ordinal + +# Attributes for a graph region + +class GraphRegionAttribute(): + + def __init__(self, colour): + self.colour = colour + +# Switch graph data region represents a task + +class SwitchGraphDataRegion(GraphDataRegion): + + def __init__(self, key, exec_comm_id, pid, tid, comm, thread_id, comm_id): + super(SwitchGraphDataRegion, self).__init__(key) + + self.title = str(pid) + " / " + str(tid) + " " + comm + # Order graph legend within exec comm by pid / tid / time + self.ordinal = str(pid).rjust(16) + str(exec_comm_id).rjust(8) + str(tid).rjust(16) + self.exec_comm_id = exec_comm_id + self.pid = pid + self.tid = tid + self.comm = comm + self.thread_id = thread_id + self.comm_id = comm_id + +# Graph data point + +class GraphDataPoint(): + + def __init__(self, data, index, x, y, altx=None, alty=None, hregion=None, vregion=None): + self.data = data + self.index = index + self.x = x + self.y = y + self.altx = altx + self.alty = alty + self.hregion = hregion + self.vregion = vregion + +# Graph data (single graph) base class + +class GraphData(object): + + def __init__(self, collection, xbase=Decimal(0), ybase=Decimal(0)): + self.collection = collection + self.points = [] + self.xbase = xbase + self.ybase = ybase + self.title = "" + + def AddPoint(self, x, y, altx=None, alty=None, hregion=None, vregion=None): + index = len(self.points) + + x = float(Decimal(x) - self.xbase) + y = float(Decimal(y) - self.ybase) + + self.points.append(GraphDataPoint(self, index, x, y, altx, alty, hregion, vregion)) + + def XToData(self, x): + return Decimal(x) + self.xbase + + def YToData(self, y): + return Decimal(y) + self.ybase + +# Switch graph data (for one CPU) + +class SwitchGraphData(GraphData): + + def __init__(self, db, collection, cpu, xbase): + super(SwitchGraphData, self).__init__(collection, xbase) + + self.cpu = cpu + self.title = "CPU " + str(cpu) + self.SelectSwitches(db) + + def SelectComms(self, db, thread_id, last_comm_id, start_time, end_time): + query = QSqlQuery(db) + QueryExec(query, "SELECT id, c_time" + " FROM comms" + " WHERE c_thread_id = " + str(thread_id) + + " AND exec_flag = " + self.collection.glb.dbref.TRUE + + " AND c_time >= " + str(start_time) + + " AND c_time <= " + str(end_time) + + " ORDER BY c_time, id") + while query.next(): + comm_id = query.value(0) + if comm_id == last_comm_id: + continue + time = query.value(1) + hregion = self.HRegion(db, thread_id, comm_id, time) + self.AddPoint(time, 1000, None, None, hregion) + + def SelectSwitches(self, db): + last_time = None + last_comm_id = None + last_thread_id = None + query = QSqlQuery(db) + QueryExec(query, "SELECT time, thread_out_id, thread_in_id, comm_out_id, comm_in_id, flags" + " FROM context_switches" + " WHERE machine_id = " + str(self.collection.machine_id) + + " AND cpu = " + str(self.cpu) + + " ORDER BY time, id") + while query.next(): + flags = int(query.value(5)) + if flags & 1: + # Schedule-out: detect and add exec's + if last_thread_id == query.value(1) and last_comm_id is not None and last_comm_id != query.value(3): + self.SelectComms(db, last_thread_id, last_comm_id, last_time, query.value(0)) + continue + # Schedule-in: add data point + if len(self.points) == 0: + start_time = self.collection.glb.StartTime(self.collection.machine_id) + hregion = self.HRegion(db, query.value(1), query.value(3), start_time) + self.AddPoint(start_time, 1000, None, None, hregion) + time = query.value(0) + comm_id = query.value(4) + thread_id = query.value(2) + hregion = self.HRegion(db, thread_id, comm_id, time) + self.AddPoint(time, 1000, None, None, hregion) + last_time = time + last_comm_id = comm_id + last_thread_id = thread_id + + def NewHRegion(self, db, key, thread_id, comm_id, time): + exec_comm_id = ExecComm(db, thread_id, time) + query = QSqlQuery(db) + QueryExec(query, "SELECT pid, tid FROM threads WHERE id = " + str(thread_id)) + if query.next(): + pid = query.value(0) + tid = query.value(1) + else: + pid = -1 + tid = -1 + query = QSqlQuery(db) + QueryExec(query, "SELECT comm FROM comms WHERE id = " + str(comm_id)) + if query.next(): + comm = query.value(0) + else: + comm = "" + return SwitchGraphDataRegion(key, exec_comm_id, pid, tid, comm, thread_id, comm_id) + + def HRegion(self, db, thread_id, comm_id, time): + key = str(thread_id) + ":" + str(comm_id) + hregion = self.collection.LookupHRegion(key) + if hregion is None: + hregion = self.NewHRegion(db, key, thread_id, comm_id, time) + self.collection.AddHRegion(key, hregion) + return hregion + +# Graph data collection (multiple related graphs) base class + +class GraphDataCollection(object): + + def __init__(self, glb): + self.glb = glb + self.data = [] + self.hregions = {} + self.xrangelo = None + self.xrangehi = None + self.yrangelo = None + self.yrangehi = None + self.dp = XY(0, 0) + + def AddGraphData(self, data): + self.data.append(data) + + def LookupHRegion(self, key): + if key in self.hregions: + return self.hregions[key] + return None + + def AddHRegion(self, key, hregion): + self.hregions[key] = hregion + +# Switch graph data collection (SwitchGraphData for each CPU) + +class SwitchGraphDataCollection(GraphDataCollection): + + def __init__(self, glb, db, machine_id): + super(SwitchGraphDataCollection, self).__init__(glb) + + self.machine_id = machine_id + self.cpus = self.SelectCPUs(db) + + self.xrangelo = glb.StartTime(machine_id) + self.xrangehi = glb.FinishTime(machine_id) + + self.yrangelo = Decimal(0) + self.yrangehi = Decimal(1000) + + for cpu in self.cpus: + self.AddGraphData(SwitchGraphData(db, self, cpu, self.xrangelo)) + + def SelectCPUs(self, db): + cpus = [] + query = QSqlQuery(db) + QueryExec(query, "SELECT DISTINCT cpu" + " FROM context_switches" + " WHERE machine_id = " + str(self.machine_id)) + while query.next(): + cpus.append(int(query.value(0))) + return sorted(cpus) + +# Switch graph data graphics item displays the graphed data + +class SwitchGraphDataGraphicsItem(QGraphicsItem): + + def __init__(self, data, graph_width, graph_height, attrs, event_handler, parent=None): + super(SwitchGraphDataGraphicsItem, self).__init__(parent) + + self.data = data + self.graph_width = graph_width + self.graph_height = graph_height + self.attrs = attrs + self.event_handler = event_handler + self.setAcceptHoverEvents(True) + + def boundingRect(self): + return QRectF(0, 0, self.graph_width, self.graph_height) + + def PaintPoint(self, painter, last, x): + if not(last is None or last.hregion.pid == 0 or x < self.attrs.subrange.x.lo): + if last.x < self.attrs.subrange.x.lo: + x0 = self.attrs.subrange.x.lo + else: + x0 = last.x + if x > self.attrs.subrange.x.hi: + x1 = self.attrs.subrange.x.hi + else: + x1 = x - 1 + x0 = self.attrs.XToPixel(x0) + x1 = self.attrs.XToPixel(x1) + + y0 = self.attrs.YToPixel(last.y) + + colour = self.attrs.region_attributes[last.hregion.key].colour + + width = x1 - x0 + 1 + if width < 2: + painter.setPen(colour) + painter.drawLine(x0, self.graph_height - y0, x0, self.graph_height) + else: + painter.fillRect(x0, self.graph_height - y0, width, self.graph_height - 1, colour) + + def paint(self, painter, _option, _widget): + last = None + for point in self.data.points: + self.PaintPoint(painter, last, point.x) + if point.x > self.attrs.subrange.x.hi: + break + last = point + self.PaintPoint(painter, last, self.attrs.subrange.x.hi + 1) + + def BinarySearchPoint(self, target): + lower_pos = 0 + higher_pos = len(self.data.points) + while True: + pos = int((lower_pos + higher_pos) / 2) + val = self.data.points[pos].x + if target >= val: + lower_pos = pos + else: + higher_pos = pos + if higher_pos <= lower_pos + 1: + return lower_pos + + def XPixelToData(self, x): + x = self.attrs.PixelToX(x) + if x < self.data.points[0].x: + x = 0 + pos = 0 + low = True + else: + pos = self.BinarySearchPoint(x) + low = False + return (low, pos, self.data.XToData(x)) + + def EventToData(self, event): + no_data = (None,) * 4 + if len(self.data.points) < 1: + return no_data + x = event.pos().x() + if x < 0: + return no_data + _low0, pos0, time_from = self.XPixelToData(x) + low1, pos1, time_to = self.XPixelToData(x + 1) + hregions = set() + hregion_times = [] + if not low1: + for i in range(pos0, pos1 + 1): + hregion = self.data.points[i].hregion + hregions.add(hregion) + if i == pos0: + time = time_from + else: + time = self.data.XToData(self.data.points[i].x) + hregion_times.append((hregion, time)) + return (time_from, time_to, hregions, hregion_times) + + def hoverMoveEvent(self, event): + time_from, time_to, hregions, _hregion_times = self.EventToData(event) + if time_from is not None: + self.event_handler.PointEvent(self.data.cpu, time_from, time_to, hregions) + + def hoverLeaveEvent(self, _event): + self.event_handler.NoPointEvent() + + def mousePressEvent(self, event): + if event.button() != Qt.RightButton: + super(SwitchGraphDataGraphicsItem, self).mousePressEvent(event) + return + _time_from, _time_to, _hregions, hregion_times = self.EventToData(event) + if hregion_times: + self.event_handler.RightClickEvent(self.data.cpu, hregion_times, event.screenPos()) + +# X-axis graphics item + +class XAxisGraphicsItem(QGraphicsItem): + + def __init__(self, width, parent=None): + super(XAxisGraphicsItem, self).__init__(parent) + + self.width = width + self.max_mark_sz = 4 + self.height = self.max_mark_sz + 1 + + def boundingRect(self): + return QRectF(0, 0, self.width, self.height) + + def Step(self): + attrs = self.parentItem().attrs + subrange = attrs.subrange.x + t = subrange.hi - subrange.lo + s = (3.0 * t) / self.width + n = 1.0 + while s > n: + n = n * 10.0 + return n + + def PaintMarks(self, painter, at_y, lo, hi, step, i): + attrs = self.parentItem().attrs + x = lo + while x <= hi: + xp = attrs.XToPixel(x) + if i % 10: + if i % 5: + sz = 1 + else: + sz = 2 + else: + sz = self.max_mark_sz + i = 0 + painter.drawLine(xp, at_y, xp, at_y + sz) + x += step + i += 1 + + def paint(self, painter, _option, _widget): + # Using QPainter::drawLine(int x1, int y1, int x2, int y2) so x2 = width -1 + painter.drawLine(0, 0, self.width - 1, 0) + n = self.Step() + attrs = self.parentItem().attrs + subrange = attrs.subrange.x + if subrange.lo: + x_offset = n - (subrange.lo % n) + else: + x_offset = 0.0 + x = subrange.lo + x_offset + i = (x / n) % 10 + self.PaintMarks(painter, 0, x, subrange.hi, n, i) + + def ScaleDimensions(self): + n = self.Step() + attrs = self.parentItem().attrs + lo = attrs.subrange.x.lo + hi = (n * 10.0) + lo + width = attrs.XToPixel(hi) + if width > 500: + width = 0 + return (n, lo, hi, width) + + def PaintScale(self, painter, at_x, at_y): + n, lo, hi, width = self.ScaleDimensions() + if not width: + return + painter.drawLine(at_x, at_y, at_x + width, at_y) + self.PaintMarks(painter, at_y, lo, hi, n, 0) + + def ScaleWidth(self): + _n, _lo, _hi, width = self.ScaleDimensions() + return width + + def ScaleHeight(self): + return self.height + + def ScaleUnit(self): + return self.Step() * 10 + +# Scale graphics item base class + +class ScaleGraphicsItem(QGraphicsItem): + + def __init__(self, axis, parent=None): + super(ScaleGraphicsItem, self).__init__(parent) + self.axis = axis + + def boundingRect(self): + scale_width = self.axis.ScaleWidth() + if not scale_width: + return QRectF() + return QRectF(0, 0, self.axis.ScaleWidth() + 100, self.axis.ScaleHeight()) + + def paint(self, painter, _option, _widget): + scale_width = self.axis.ScaleWidth() + if not scale_width: + return + self.axis.PaintScale(painter, 0, 5) + x = scale_width + 4 + painter.drawText(QPointF(x, 10), self.Text()) + + def Unit(self): + return self.axis.ScaleUnit() + + def Text(self): + return "" + +# Switch graph scale graphics item + +class SwitchScaleGraphicsItem(ScaleGraphicsItem): + + def Text(self): + unit = self.Unit() + if unit >= 1000000000: + unit = int(unit / 1000000000) + us = "s" + elif unit >= 1000000: + unit = int(unit / 1000000) + us = "ms" + elif unit >= 1000: + unit = int(unit / 1000) + us = "us" + else: + unit = int(unit) + us = "ns" + return " = " + str(unit) + " " + us + +# Switch graph graphics item contains graph title, scale, x/y-axis, and the graphed data + +class SwitchGraphGraphicsItem(QGraphicsItem): + + def __init__(self, collection, data, attrs, event_handler, first, parent=None): + super(SwitchGraphGraphicsItem, self).__init__(parent) + self.collection = collection + self.data = data + self.attrs = attrs + self.event_handler = event_handler + + margin = 20 + title_width = 50 + + self.title_graphics = QGraphicsSimpleTextItem(data.title, self) + + self.title_graphics.setPos(margin, margin) + graph_width = attrs.XToPixel(attrs.subrange.x.hi) + 1 + graph_height = attrs.YToPixel(attrs.subrange.y.hi) + 1 + + self.graph_origin_x = margin + title_width + margin + self.graph_origin_y = graph_height + margin + + _x_axis_size = 1 + y_axis_size = 1 + self.yline = QGraphicsLineItem(0, 0, 0, graph_height, self) + + self.x_axis = XAxisGraphicsItem(graph_width, self) + self.x_axis.setPos(self.graph_origin_x, self.graph_origin_y + 1) + + if first: + self.scale_item = SwitchScaleGraphicsItem(self.x_axis, self) + self.scale_item.setPos(self.graph_origin_x, self.graph_origin_y + 10) + + self.yline.setPos(self.graph_origin_x - y_axis_size, self.graph_origin_y - graph_height) + + self.axis_point = QGraphicsLineItem(0, 0, 0, 0, self) + self.axis_point.setPos(self.graph_origin_x - 1, self.graph_origin_y +1) + + self.width = self.graph_origin_x + graph_width + margin + self.height = self.graph_origin_y + margin + + self.graph = SwitchGraphDataGraphicsItem(data, graph_width, graph_height, attrs, event_handler, self) + self.graph.setPos(self.graph_origin_x, self.graph_origin_y - graph_height) + + if parent and 'EnableRubberBand' in dir(parent): + parent.EnableRubberBand(self.graph_origin_x, self.graph_origin_x + graph_width - 1, self) + + def boundingRect(self): + return QRectF(0, 0, self.width, self.height) + + def paint(self, painter, option, widget): + pass + + def RBXToPixel(self, x): + return self.attrs.PixelToX(x - self.graph_origin_x) + + def RBXRangeToPixel(self, x0, x1): + return (self.RBXToPixel(x0), self.RBXToPixel(x1 + 1)) + + def RBPixelToTime(self, x): + if x < self.data.points[0].x: + return self.data.XToData(0) + return self.data.XToData(x) + + def RBEventTimes(self, x0, x1): + x0, x1 = self.RBXRangeToPixel(x0, x1) + time_from = self.RBPixelToTime(x0) + time_to = self.RBPixelToTime(x1) + return (time_from, time_to) + + def RBEvent(self, x0, x1): + time_from, time_to = self.RBEventTimes(x0, x1) + self.event_handler.RangeEvent(time_from, time_to) + + def RBMoveEvent(self, x0, x1): + if x1 < x0: + x0, x1 = x1, x0 + self.RBEvent(x0, x1) + + def RBReleaseEvent(self, x0, x1, selection_state): + if x1 < x0: + x0, x1 = x1, x0 + x0, x1 = self.RBXRangeToPixel(x0, x1) + self.event_handler.SelectEvent(x0, x1, selection_state) + +# Graphics item to draw a vertical bracket (used to highlight "forward" sub-range) + +class VerticalBracketGraphicsItem(QGraphicsItem): + + def __init__(self, parent=None): + super(VerticalBracketGraphicsItem, self).__init__(parent) + + self.width = 0 + self.height = 0 + self.hide() + + def SetSize(self, width, height): + self.width = width + 1 + self.height = height + 1 + + def boundingRect(self): + return QRectF(0, 0, self.width, self.height) + + def paint(self, painter, _option, _widget): + colour = QColor(255, 255, 0, 32) + painter.fillRect(0, 0, self.width, self.height, colour) + x1 = self.width - 1 + y1 = self.height - 1 + painter.drawLine(0, 0, x1, 0) + painter.drawLine(0, 0, 0, 3) + painter.drawLine(x1, 0, x1, 3) + painter.drawLine(0, y1, x1, y1) + painter.drawLine(0, y1, 0, y1 - 3) + painter.drawLine(x1, y1, x1, y1 - 3) + +# Graphics item to contain graphs arranged vertically + +class VertcalGraphSetGraphicsItem(QGraphicsItem): + + def __init__(self, collection, attrs, event_handler, child_class, parent=None): + super(VertcalGraphSetGraphicsItem, self).__init__(parent) + + self.collection = collection + + self.top = 10 + + self.width = 0 + self.height = self.top + + self.rubber_band = None + self.rb_enabled = False + self.rb_in_view = False + self.rb_xlo = 0 + self.rb_xhi = 0 + self.rb_event_handler = None + self.rb_origin = None + + first = True + for data in collection.data: + child = child_class(collection, data, attrs, event_handler, first, self) + child.setPos(0, self.height + 1) + rect = child.boundingRect() + if rect.right() > self.width: + self.width = rect.right() + self.height = self.height + rect.bottom() + 1 + first = False + + self.bracket = VerticalBracketGraphicsItem(self) + + def EnableRubberBand(self, xlo, xhi, rb_event_handler): + if self.rb_enabled: + return + self.rb_enabled = True + self.rb_in_view = False + self.setAcceptedMouseButtons(Qt.LeftButton) + self.rb_xlo = xlo + self.rb_xhi = xhi + self.rb_event_handler = rb_event_handler + + def mousePressEvent(self, event): + if self.rb_enabled: + self.MousePressEvent(event) + + def mouseMoveEvent(self, event): + if self.rb_enabled: + self.MouseMoveEvent(event) + + def mouseReleaseEvent(self, event): + if self.rb_enabled: + self.MouseReleaseEvent(event) + + def boundingRect(self): + return QRectF(0, 0, self.width, self.height) + + def paint(self, painter, option, widget): + pass + + def RubberBandParent(self): + scene = self.scene() + view = scene.views()[0] + viewport = view.viewport() + return viewport + + def RubberBandSetGeometry(self, rect): + scene_rectf = self.mapRectToScene(QRectF(rect)) + scene = self.scene() + view = scene.views()[0] + poly = view.mapFromScene(scene_rectf) + self.rubber_band.setGeometry(poly.boundingRect()) + + def SetSelection(self, selection_state): + if self.rubber_band: + if selection_state: + self.RubberBandSetGeometry(selection_state) + self.rubber_band.show() + else: + self.rubber_band.hide() + + def SetBracket(self, rect): + if rect: + x, y, width, height = rect.x(), rect.y(), rect.width(), rect.height() + self.bracket.setPos(x, y) + self.bracket.SetSize(width, height) + self.bracket.show() + else: + self.bracket.hide() + + def RubberBandX(self, event): + x = event.pos().toPoint().x() + if x < self.rb_xlo: + x = self.rb_xlo + elif x > self.rb_xhi: + x = self.rb_xhi + else: + self.rb_in_view = True + return x + + def RubberBandRect(self, x): + if self.rb_origin.x() <= x: + width = x - self.rb_origin.x() + rect = QRect(self.rb_origin, QSize(width, self.height)) + else: + width = self.rb_origin.x() - x + top_left = QPoint(self.rb_origin.x() - width, self.rb_origin.y()) + rect = QRect(top_left, QSize(width, self.height)) + return rect + + def MousePressEvent(self, event): + self.rb_in_view = False + x = self.RubberBandX(event) + self.rb_origin = QPoint(x, self.top) + if self.rubber_band is None: + self.rubber_band = QRubberBand(QRubberBand.Rectangle, self.RubberBandParent()) + self.RubberBandSetGeometry(QRect(self.rb_origin, QSize(0, self.height))) + if self.rb_in_view: + self.rubber_band.show() + self.rb_event_handler.RBMoveEvent(x, x) + else: + self.rubber_band.hide() + + def MouseMoveEvent(self, event): + x = self.RubberBandX(event) + rect = self.RubberBandRect(x) + self.RubberBandSetGeometry(rect) + if self.rb_in_view: + self.rubber_band.show() + self.rb_event_handler.RBMoveEvent(self.rb_origin.x(), x) + + def MouseReleaseEvent(self, event): + x = self.RubberBandX(event) + if self.rb_in_view: + selection_state = self.RubberBandRect(x) + else: + selection_state = None + self.rb_event_handler.RBReleaseEvent(self.rb_origin.x(), x, selection_state) + +# Switch graph legend data model + +class SwitchGraphLegendModel(QAbstractTableModel): + + def __init__(self, collection, region_attributes, parent=None): + super(SwitchGraphLegendModel, self).__init__(parent) + + self.region_attributes = region_attributes + + self.child_items = sorted(collection.hregions.values(), key=GraphDataRegionOrdinal) + self.child_count = len(self.child_items) + + self.highlight_set = set() + + self.column_headers = ("pid", "tid", "comm") + + def rowCount(self, _parent): + return self.child_count + + def headerData(self, section, orientation, role): + if role != Qt.DisplayRole: + return None + if orientation != Qt.Horizontal: + return None + return self.columnHeader(section) + + def index(self, row, column, _parent): + return self.createIndex(row, column, self.child_items[row]) + + def columnCount(self, _parent=None): + return len(self.column_headers) + + def columnHeader(self, column): + return self.column_headers[column] + + def data(self, index, role): + if role == Qt.BackgroundRole: + child = self.child_items[index.row()] + if child in self.highlight_set: + return self.region_attributes[child.key].colour + return None + if role == Qt.ForegroundRole: + child = self.child_items[index.row()] + if child in self.highlight_set: + return QColor(255, 255, 255) + return self.region_attributes[child.key].colour + if role != Qt.DisplayRole: + return None + hregion = self.child_items[index.row()] + col = index.column() + if col == 0: + return hregion.pid + if col == 1: + return hregion.tid + if col == 2: + return hregion.comm + return None + + def SetHighlight(self, row, _set_highlight): + child = self.child_items[row] + top_left = self.createIndex(row, 0, child) + bottom_right = self.createIndex(row, len(self.column_headers) - 1, child) + self.dataChanged.emit(top_left, bottom_right) + + def Highlight(self, highlight_set): + for row in range(self.child_count): + child = self.child_items[row] + if child in self.highlight_set: + if child not in highlight_set: + self.SetHighlight(row, False) + elif child in highlight_set: + self.SetHighlight(row, True) + self.highlight_set = highlight_set + +# Switch graph legend is a table + +class SwitchGraphLegend(QWidget): + + def __init__(self, collection, region_attributes, parent=None): + super(SwitchGraphLegend, self).__init__(parent) + + self.data_model = SwitchGraphLegendModel(collection, region_attributes) + + self.model = QSortFilterProxyModel() + self.model.setSourceModel(self.data_model) + + self.view = QTableView() + self.view.setModel(self.model) + self.view.setEditTriggers(QAbstractItemView.NoEditTriggers) + self.view.verticalHeader().setVisible(False) + self.view.sortByColumn(-1, Qt.AscendingOrder) + self.view.setSortingEnabled(True) + self.view.resizeColumnsToContents() + self.view.resizeRowsToContents() + + self.vbox = VBoxLayout(self.view) + self.setLayout(self.vbox) + + sz1 = self.view.columnWidth(0) + self.view.columnWidth(1) + self.view.columnWidth(2) + 2 + sz1 = sz1 + self.view.verticalScrollBar().sizeHint().width() + self.saved_size = sz1 + + def resizeEvent(self, event): + self.saved_size = self.size().width() + super(SwitchGraphLegend, self).resizeEvent(event) + + def Highlight(self, highlight_set): + self.data_model.Highlight(highlight_set) + self.update() + + def changeEvent(self, event): + if event.type() == QEvent.FontChange: + self.view.resizeRowsToContents() + self.view.resizeColumnsToContents() + # Need to resize rows again after column resize + self.view.resizeRowsToContents() + super(SwitchGraphLegend, self).changeEvent(event) + +# Random colour generation + +def RGBColourTooLight(r, g, _b): + if g > 230: + return True + if g <= 160: + return False + if r <= 180 and g <= 180: + return False + if r < 60: + return False + return True + +def GenerateColours(x): + cs = [0] + for i in range(1, x): + cs.append(int((255.0 / i) + 0.5)) + colours = [] + for r in cs: + for g in cs: + for b in cs: + # Exclude black and colours that look too light against a white background + if (r, g, b) == (0, 0, 0) or RGBColourTooLight(r, g, b): + continue + colours.append(QColor(r, g, b)) + return colours + +def GenerateNColours(n): + for x in range(2, n + 2): + colours = GenerateColours(x) + if len(colours) >= n: + return colours + return [] + +def GenerateNRandomColours(n, seed): + colours = GenerateNColours(n) + random.seed(seed) + random.shuffle(colours) + return colours + +# Graph attributes, in particular the scale and subrange that change when zooming + +class GraphAttributes(): + + def __init__(self, scale, subrange, region_attributes, dp): + self.scale = scale + self.subrange = subrange + self.region_attributes = region_attributes + # Rounding avoids errors due to finite floating point precision + self.dp = dp # data decimal places + self.Update() + + def XToPixel(self, x): + return int(round((x - self.subrange.x.lo) * self.scale.x, self.pdp.x)) + + def YToPixel(self, y): + return int(round((y - self.subrange.y.lo) * self.scale.y, self.pdp.y)) + + def PixelToXRounded(self, px): + return round((round(px, 0) / self.scale.x), self.dp.x) + self.subrange.x.lo + + def PixelToYRounded(self, py): + return round((round(py, 0) / self.scale.y), self.dp.y) + self.subrange.y.lo + + def PixelToX(self, px): + x = self.PixelToXRounded(px) + if self.pdp.x == 0: + rt = self.XToPixel(x) + if rt > px: + return x - 1 + return x + + def PixelToY(self, py): + y = self.PixelToYRounded(py) + if self.pdp.y == 0: + rt = self.YToPixel(y) + if rt > py: + return y - 1 + return y + + def ToPDP(self, dp, scale): + # Calculate pixel decimal places: + # (10 ** dp) is the minimum delta in the data + # scale it to get the minimum delta in pixels + # log10 gives the number of decimals places negatively + # subtrace 1 to divide by 10 + # round to the lower negative number + # change the sign to get the number of decimals positively + x = math.log10((10 ** dp) * scale) + if x < 0: + x -= 1 + x = -int(math.floor(x) - 0.1) + else: + x = 0 + return x + + def Update(self): + x = self.ToPDP(self.dp.x, self.scale.x) + y = self.ToPDP(self.dp.y, self.scale.y) + self.pdp = XY(x, y) # pixel decimal places + +# Switch graph splitter which divides the CPU graphs from the legend + +class SwitchGraphSplitter(QSplitter): + + def __init__(self, parent=None): + super(SwitchGraphSplitter, self).__init__(parent) + + self.first_time = False + + def resizeEvent(self, event): + if self.first_time: + self.first_time = False + sz1 = self.widget(1).view.columnWidth(0) + self.widget(1).view.columnWidth(1) + self.widget(1).view.columnWidth(2) + 2 + sz1 = sz1 + self.widget(1).view.verticalScrollBar().sizeHint().width() + sz0 = self.size().width() - self.handleWidth() - sz1 + self.setSizes([sz0, sz1]) + elif not(self.widget(1).saved_size is None): + sz1 = self.widget(1).saved_size + sz0 = self.size().width() - self.handleWidth() - sz1 + self.setSizes([sz0, sz1]) + super(SwitchGraphSplitter, self).resizeEvent(event) + +# Graph widget base class + +class GraphWidget(QWidget): + + graph_title_changed = Signal(object) + + def __init__(self, parent=None): + super(GraphWidget, self).__init__(parent) + + def GraphTitleChanged(self, title): + self.graph_title_changed.emit(title) + + def Title(self): + return "" + +# Display time in s, ms, us or ns + +def ToTimeStr(val): + val = Decimal(val) + if val >= 1000000000: + return "{} s".format((val / 1000000000).quantize(Decimal("0.000000001"))) + if val >= 1000000: + return "{} ms".format((val / 1000000).quantize(Decimal("0.000001"))) + if val >= 1000: + return "{} us".format((val / 1000).quantize(Decimal("0.001"))) + return "{} ns".format(val.quantize(Decimal("1"))) + +# Switch (i.e. context switch i.e. Time Chart by CPU) graph widget which contains the CPU graphs and the legend and control buttons + +class SwitchGraphWidget(GraphWidget): + + def __init__(self, glb, collection, parent=None): + super(SwitchGraphWidget, self).__init__(parent) + + self.glb = glb + self.collection = collection + + self.back_state = [] + self.forward_state = [] + self.selection_state = (None, None) + self.fwd_rect = None + self.start_time = self.glb.StartTime(collection.machine_id) + + i = 0 + hregions = collection.hregions.values() + colours = GenerateNRandomColours(len(hregions), 1013) + region_attributes = {} + for hregion in hregions: + if hregion.pid == 0 and hregion.tid == 0: + region_attributes[hregion.key] = GraphRegionAttribute(QColor(0, 0, 0)) + else: + region_attributes[hregion.key] = GraphRegionAttribute(colours[i]) + i = i + 1 + + # Default to entire range + xsubrange = Subrange(0.0, float(collection.xrangehi - collection.xrangelo) + 1.0) + ysubrange = Subrange(0.0, float(collection.yrangehi - collection.yrangelo) + 1.0) + subrange = XY(xsubrange, ysubrange) + + scale = self.GetScaleForRange(subrange) + + self.attrs = GraphAttributes(scale, subrange, region_attributes, collection.dp) + + self.item = VertcalGraphSetGraphicsItem(collection, self.attrs, self, SwitchGraphGraphicsItem) + + self.scene = QGraphicsScene() + self.scene.addItem(self.item) + + self.view = QGraphicsView(self.scene) + self.view.centerOn(0, 0) + self.view.setAlignment(Qt.AlignLeft | Qt.AlignTop) + + self.legend = SwitchGraphLegend(collection, region_attributes) + + self.splitter = SwitchGraphSplitter() + self.splitter.addWidget(self.view) + self.splitter.addWidget(self.legend) + + self.point_label = QLabel("") + self.point_label.setSizePolicy(QSizePolicy.Preferred, QSizePolicy.Fixed) + + self.back_button = QToolButton() + self.back_button.setIcon(self.style().standardIcon(QStyle.SP_ArrowLeft)) + self.back_button.setDisabled(True) + self.back_button.released.connect(self.Back) + + self.forward_button = QToolButton() + self.forward_button.setIcon(self.style().standardIcon(QStyle.SP_ArrowRight)) + self.forward_button.setDisabled(True) + self.forward_button.released.connect(self.Forward) + + self.zoom_button = QToolButton() + self.zoom_button.setText("Zoom") + self.zoom_button.setDisabled(True) + self.zoom_button.released.connect(self.Zoom) + + self.hbox = HBoxLayout(self.back_button, self.forward_button, self.zoom_button, self.point_label) + + self.vbox = VBoxLayout(self.splitter, self.hbox) + + self.setLayout(self.vbox) + + def GetScaleForRangeX(self, xsubrange): + # Default graph 1000 pixels wide + dflt = 1000.0 + r = xsubrange.hi - xsubrange.lo + return dflt / r + + def GetScaleForRangeY(self, ysubrange): + # Default graph 50 pixels high + dflt = 50.0 + r = ysubrange.hi - ysubrange.lo + return dflt / r + + def GetScaleForRange(self, subrange): + # Default graph 1000 pixels wide, 50 pixels high + xscale = self.GetScaleForRangeX(subrange.x) + yscale = self.GetScaleForRangeY(subrange.y) + return XY(xscale, yscale) + + def PointEvent(self, cpu, time_from, _time_to, hregions): + text = "CPU: " + str(cpu) + time_from = time_from.quantize(Decimal(1)) + rel_time_from = time_from - self.glb.StartTime(self.collection.machine_id) + text = text + " Time: " + str(time_from) + " (+" + ToTimeStr(rel_time_from) + ")" + self.point_label.setText(text) + self.legend.Highlight(hregions) + + def RightClickEvent(self, _cpu, hregion_times, pos): + if not IsSelectable(self.glb.db, "calls", "WHERE parent_id >= 0"): + return + menu = QMenu(self.view) + for hregion, time in hregion_times: + thread_at_time = (hregion.exec_comm_id, hregion.thread_id, time) + menu_text = "Show Call Tree for {} {}:{} at {}".format(hregion.comm, hregion.pid, hregion.tid, time) + menu.addAction(CreateAction(menu_text, "Show Call Tree", lambda a=None, args=thread_at_time: self.RightClickSelect(args), self.view)) + menu.exec_(pos) + + def RightClickSelect(self, args): + CallTreeWindow(self.glb, self.glb.mainwindow, thread_at_time=args) + + def NoPointEvent(self): + self.point_label.setText("") + self.legend.Highlight({}) + + def RangeEvent(self, time_from, time_to): + time_from = time_from.quantize(Decimal(1)) + time_to = time_to.quantize(Decimal(1)) + if time_to <= time_from: + self.point_label.setText("") + return + rel_time_from = time_from - self.start_time + rel_time_to = time_to - self.start_time + text = " Time: " + str(time_from) + " (+" + ToTimeStr(rel_time_from) + ") to: " + str(time_to) + " (+" + ToTimeStr(rel_time_to) + ")" + text = text + " duration: " + ToTimeStr(time_to - time_from) + self.point_label.setText(text) + + def BackState(self): + return (self.attrs.subrange, self.attrs.scale, self.selection_state, self.fwd_rect) + + def PushBackState(self): + state = copy.deepcopy(self.BackState()) + self.back_state.append(state) + self.back_button.setEnabled(True) + + def PopBackState(self): + self.attrs.subrange, self.attrs.scale, self.selection_state, self.fwd_rect = self.back_state.pop() + self.attrs.Update() + if not self.back_state: + self.back_button.setDisabled(True) + + def PushForwardState(self): + state = copy.deepcopy(self.BackState()) + self.forward_state.append(state) + self.forward_button.setEnabled(True) + + def PopForwardState(self): + self.attrs.subrange, self.attrs.scale, self.selection_state, self.fwd_rect = self.forward_state.pop() + self.attrs.Update() + if not self.forward_state: + self.forward_button.setDisabled(True) + + def Title(self): + time_from = self.collection.xrangelo + Decimal(self.attrs.subrange.x.lo) + time_to = self.collection.xrangelo + Decimal(self.attrs.subrange.x.hi) + rel_time_from = time_from - self.start_time + rel_time_to = time_to - self.start_time + title = "+" + ToTimeStr(rel_time_from) + " to +" + ToTimeStr(rel_time_to) + title = title + " (" + ToTimeStr(time_to - time_from) + ")" + return title + + def Update(self): + selected_subrange, selection_state = self.selection_state + self.item.SetSelection(selection_state) + self.item.SetBracket(self.fwd_rect) + self.zoom_button.setDisabled(selected_subrange is None) + self.GraphTitleChanged(self.Title()) + self.item.update(self.item.boundingRect()) + + def Back(self): + if not self.back_state: + return + self.PushForwardState() + self.PopBackState() + self.Update() + + def Forward(self): + if not self.forward_state: + return + self.PushBackState() + self.PopForwardState() + self.Update() + + def SelectEvent(self, x0, x1, selection_state): + if selection_state is None: + selected_subrange = None + else: + if x1 - x0 < 1.0: + x1 += 1.0 + selected_subrange = Subrange(x0, x1) + self.selection_state = (selected_subrange, selection_state) + self.zoom_button.setDisabled(selected_subrange is None) + + def Zoom(self): + selected_subrange, selection_state = self.selection_state + if selected_subrange is None: + return + self.fwd_rect = selection_state + self.item.SetSelection(None) + self.PushBackState() + self.attrs.subrange.x = selected_subrange + self.forward_state = [] + self.forward_button.setDisabled(True) + self.selection_state = (None, None) + self.fwd_rect = None + self.attrs.scale.x = self.GetScaleForRangeX(self.attrs.subrange.x) + self.attrs.Update() + self.Update() + +# Slow initialization - perform non-GUI initialization in a separate thread and put up a modal message box while waiting + +class SlowInitClass(): + + def __init__(self, glb, title, init_fn): + self.init_fn = init_fn + self.done = False + self.result = None + + self.msg_box = QMessageBox(glb.mainwindow) + self.msg_box.setText("Initializing " + title + ". Please wait.") + self.msg_box.setWindowTitle("Initializing " + title) + self.msg_box.setWindowIcon(glb.mainwindow.style().standardIcon(QStyle.SP_MessageBoxInformation)) + + self.init_thread = Thread(self.ThreadFn, glb) + self.init_thread.done.connect(self.Done, Qt.QueuedConnection) + + self.init_thread.start() + + def Done(self): + self.msg_box.done(0) + + def ThreadFn(self, glb): + conn_name = "SlowInitClass" + str(os.getpid()) + db, _dbname = glb.dbref.Open(conn_name) + self.result = self.init_fn(db) + self.done = True + return (True, 0) + + def Result(self): + while not self.done: + self.msg_box.exec_() + self.init_thread.wait() + return self.result + +def SlowInit(glb, title, init_fn): + init = SlowInitClass(glb, title, init_fn) + return init.Result() + +# Time chart by CPU window + +class TimeChartByCPUWindow(QMdiSubWindow): + + def __init__(self, glb, parent=None): + super(TimeChartByCPUWindow, self).__init__(parent) + + self.glb = glb + self.machine_id = glb.HostMachineId() + self.collection_name = "SwitchGraphDataCollection " + str(self.machine_id) + + collection = LookupModel(self.collection_name) + if collection is None: + collection = SlowInit(glb, "Time Chart", self.Init) + + self.widget = SwitchGraphWidget(glb, collection, self) + self.view = self.widget + + self.base_title = "Time Chart by CPU" + self.setWindowTitle(self.base_title + self.widget.Title()) + self.widget.graph_title_changed.connect(self.GraphTitleChanged) + + self.setWidget(self.widget) + + AddSubWindow(glb.mainwindow.mdi_area, self, self.windowTitle()) + + def Init(self, db): + return LookupCreateModel(self.collection_name, lambda : SwitchGraphDataCollection(self.glb, db, self.machine_id)) + + def GraphTitleChanged(self, title): + self.setWindowTitle(self.base_title + " : " + title) + +# Child data item finder + +class ChildDataItemFinder(): + + def __init__(self, root): + self.root = root + self.value, self.direction, self.pattern, self.last_value, self.last_pattern = (None,) * 5 + self.rows = [] + self.pos = 0 + + def FindSelect(self): + self.rows = [] + if self.pattern: + pattern = re.compile(self.value) + for child in self.root.child_items: + for column_data in child.data: + if re.search(pattern, str(column_data)) is not None: + self.rows.append(child.row) + break + else: + for child in self.root.child_items: + for column_data in child.data: + if self.value in str(column_data): + self.rows.append(child.row) + break + + def FindValue(self): + self.pos = 0 + if self.last_value != self.value or self.pattern != self.last_pattern: + self.FindSelect() + if not len(self.rows): + return -1 + return self.rows[self.pos] + + def FindThread(self): + if self.direction == 0 or self.value != self.last_value or self.pattern != self.last_pattern: + row = self.FindValue() + elif len(self.rows): + if self.direction > 0: + self.pos += 1 + if self.pos >= len(self.rows): + self.pos = 0 + else: + self.pos -= 1 + if self.pos < 0: + self.pos = len(self.rows) - 1 + row = self.rows[self.pos] + else: + row = -1 + return (True, row) + + def Find(self, value, direction, pattern, _context, callback): + self.value, self.direction, self.pattern, self.last_value, self.last_pattern = (value, direction,pattern, self.value, self.pattern) + # Use a thread so the UI is not blocked + thread = Thread(self.FindThread) + thread.done.connect(lambda row, t=thread, c=callback: self.FindDone(t, c, row), Qt.QueuedConnection) + thread.start() + + def FindDone(self, _thread, callback, row): + callback(row) + +# Number of database records to fetch in one go + +glb_chunk_sz = 10000 + +# Background process for SQL data fetcher + +class SQLFetcherProcess(): + + def __init__(self, dbref, sql, buffer, head, tail, fetch_count, fetching_done, process_target, wait_event, fetched_event, prep): + # Need a unique connection name + conn_name = "SQLFetcher" + str(os.getpid()) + self.db, _dbname = dbref.Open(conn_name) + self.sql = sql + self.buffer = buffer + self.head = head + self.tail = tail + self.fetch_count = fetch_count + self.fetching_done = fetching_done + self.process_target = process_target + self.wait_event = wait_event + self.fetched_event = fetched_event + self.prep = prep + self.query = QSqlQuery(self.db) + self.query_limit = 0 if "$$last_id$$" in sql else 2 + self.last_id = -1 + self.fetched = 0 + self.more = True + self.local_head = self.head.value + self.local_tail = self.tail.value + + def Select(self): + if self.query_limit: + if self.query_limit == 1: + return + self.query_limit -= 1 + stmt = self.sql.replace("$$last_id$$", str(self.last_id)) + QueryExec(self.query, stmt) + + def Next(self): + if not self.query.next(): + self.Select() + if not self.query.next(): + return None + self.last_id = self.query.value(0) + return self.prep(self.query) + + def WaitForTarget(self): + while True: + self.wait_event.clear() + target = self.process_target.value + if target > self.fetched or target < 0: + break + self.wait_event.wait() + return target + + def HasSpace(self, sz): + if self.local_tail <= self.local_head: + space = len(self.buffer) - self.local_head + if space > sz: + return True + if space >= glb_nsz: + # Use 0 (or space < glb_nsz) to mean there is no more at the top of the buffer + nd = pickle.dumps(0, pickle.HIGHEST_PROTOCOL) + self.buffer[self.local_head : self.local_head + len(nd)] = nd + self.local_head = 0 + if self.local_tail - self.local_head > sz: + return True + return False + + def WaitForSpace(self, sz): + if self.HasSpace(sz): + return + while True: + self.wait_event.clear() + self.local_tail = self.tail.value + if self.HasSpace(sz): + return + self.wait_event.wait() + + def AddToBuffer(self, obj): + d = pickle.dumps(obj, pickle.HIGHEST_PROTOCOL) + n = len(d) + nd = pickle.dumps(n, pickle.HIGHEST_PROTOCOL) + sz = n + glb_nsz + self.WaitForSpace(sz) + pos = self.local_head + self.buffer[pos : pos + len(nd)] = nd + self.buffer[pos + glb_nsz : pos + sz] = d + self.local_head += sz + + def FetchBatch(self, batch_size): + fetched = 0 + while batch_size > fetched: + obj = self.Next() + if obj is None: + self.more = False + break + self.AddToBuffer(obj) + fetched += 1 + if fetched: + self.fetched += fetched + with self.fetch_count.get_lock(): + self.fetch_count.value += fetched + self.head.value = self.local_head + self.fetched_event.set() + + def Run(self): + while self.more: + target = self.WaitForTarget() + if target < 0: + break + batch_size = min(glb_chunk_sz, target - self.fetched) + self.FetchBatch(batch_size) + self.fetching_done.value = True + self.fetched_event.set() + +def SQLFetcherFn(*x): + process = SQLFetcherProcess(*x) + process.Run() + +# SQL data fetcher + +class SQLFetcher(QObject): + + done = Signal(object) + + def __init__(self, glb, sql, prep, process_data, parent=None): + super(SQLFetcher, self).__init__(parent) + self.process_data = process_data + self.more = True + self.target = 0 + self.last_target = 0 + self.fetched = 0 + self.buffer_size = 16 * 1024 * 1024 + self.buffer = Array(c_char, self.buffer_size, lock=False) + self.head = Value(c_longlong) + self.tail = Value(c_longlong) + self.local_tail = 0 + self.fetch_count = Value(c_longlong) + self.fetching_done = Value(c_bool) + self.last_count = 0 + self.process_target = Value(c_longlong) + self.wait_event = Event() + self.fetched_event = Event() + glb.AddInstanceToShutdownOnExit(self) + self.process = Process(target=SQLFetcherFn, args=(glb.dbref, sql, self.buffer, self.head, self.tail, self.fetch_count, self.fetching_done, self.process_target, self.wait_event, self.fetched_event, prep)) + self.process.start() + self.thread = Thread(self.Thread) + self.thread.done.connect(self.ProcessData, Qt.QueuedConnection) + self.thread.start() + + def Shutdown(self): + # Tell the thread and process to exit + self.process_target.value = -1 + self.wait_event.set() + self.more = False + self.fetching_done.value = True + self.fetched_event.set() + + def Thread(self): + if not self.more: + return True, 0 + while True: + self.fetched_event.clear() + fetch_count = self.fetch_count.value + if fetch_count != self.last_count: + break + if self.fetching_done.value: + self.more = False + return True, 0 + self.fetched_event.wait() + count = fetch_count - self.last_count + self.last_count = fetch_count + self.fetched += count + return False, count + + def Fetch(self, nr): + if not self.more: + # -1 inidcates there are no more + return -1 + result = self.fetched + extra = result + nr - self.target + if extra > 0: + self.target += extra + # process_target < 0 indicates shutting down + if self.process_target.value >= 0: + self.process_target.value = self.target + self.wait_event.set() + return result + + def RemoveFromBuffer(self): + pos = self.local_tail + if len(self.buffer) - pos < glb_nsz: + pos = 0 + n = pickle.loads(self.buffer[pos : pos + glb_nsz]) + if n == 0: + pos = 0 + n = pickle.loads(self.buffer[0 : glb_nsz]) + pos += glb_nsz + obj = pickle.loads(self.buffer[pos : pos + n]) + self.local_tail = pos + n + return obj + + def ProcessData(self, count): + for _i in range(count): + obj = self.RemoveFromBuffer() + self.process_data(obj) + self.tail.value = self.local_tail + self.wait_event.set() + self.done.emit(count) + +# Fetch more records bar + +class FetchMoreRecordsBar(): + + def __init__(self, model, parent): + self.model = model + + self.label = QLabel("Number of records (x " + "{:,}".format(glb_chunk_sz) + ") to fetch:") + self.label.setSizePolicy(QSizePolicy.Fixed, QSizePolicy.Fixed) + + self.fetch_count = QSpinBox() + self.fetch_count.setRange(1, 1000000) + self.fetch_count.setValue(10) + self.fetch_count.setSizePolicy(QSizePolicy.Fixed, QSizePolicy.Fixed) + + self.fetch = QPushButton("Go!") + self.fetch.setSizePolicy(QSizePolicy.Fixed, QSizePolicy.Fixed) + self.fetch.released.connect(self.FetchMoreRecords) + + self.progress = QProgressBar() + self.progress.setRange(0, 100) + self.progress.hide() + + self.done_label = QLabel("All records fetched") + self.done_label.hide() + + self.spacer = QLabel("") + + self.close_button = QToolButton() + self.close_button.setIcon(parent.style().standardIcon(QStyle.SP_DockWidgetCloseButton)) + self.close_button.released.connect(self.Deactivate) + + self.hbox = QHBoxLayout() + self.hbox.setContentsMargins(0, 0, 0, 0) + + self.hbox.addWidget(self.label) + self.hbox.addWidget(self.fetch_count) + self.hbox.addWidget(self.fetch) + self.hbox.addWidget(self.spacer) + self.hbox.addWidget(self.progress) + self.hbox.addWidget(self.done_label) + self.hbox.addWidget(self.close_button) + + self.bar = QWidget() + self.bar.setLayout(self.hbox) + self.bar.show() + + self.in_progress = False + self.start = 0 + self.model.progress.connect(self.Progress) + + self.done = False + + if not model.HasMoreRecords(): + self.Done() + + def Widget(self): + return self.bar + + def Activate(self): + self.bar.show() + self.fetch.setFocus() + + def Deactivate(self): + self.bar.hide() + + def Enable(self, enable): + self.fetch.setEnabled(enable) + self.fetch_count.setEnabled(enable) + + def Busy(self): + self.Enable(False) + self.fetch.hide() + self.spacer.hide() + self.progress.show() + + def Idle(self): + self.in_progress = False + self.Enable(True) + self.progress.hide() + self.fetch.show() + self.spacer.show() + + def Target(self): + return self.fetch_count.value() * glb_chunk_sz + + def Done(self): + self.done = True + self.Idle() + self.label.hide() + self.fetch_count.hide() + self.fetch.hide() + self.spacer.hide() + self.done_label.show() + + def Progress(self, count): + if self.in_progress: + if count: + percent = ((count - self.start) * 100) / self.Target() + if percent >= 100: + self.Idle() + else: + self.progress.setValue(percent) + if not count: + # Count value of zero means no more records + self.Done() + + def FetchMoreRecords(self): + if self.done: + return + self.progress.setValue(0) + self.Busy() + self.in_progress = True + self.start = self.model.FetchMoreRecords(self.Target()) + +# Brance data model level two item + +class BranchLevelTwoItem(): + + def __init__(self, row, col, text, parent_item): + self.row = row + self.parent_item = parent_item + self.data = [""] * (col + 1) + self.data[col] = text + self.level = 2 + + def getParentItem(self): + return self.parent_item + + def getRow(self): + return self.row + + def childCount(self): + return 0 + + def hasChildren(self): + return False + + def getData(self, column): + return self.data[column] + +# Brance data model level one item + +class BranchLevelOneItem(): + + def __init__(self, glb, row, data, parent_item): + self.glb = glb + self.row = row + self.parent_item = parent_item + self.child_count = 0 + self.child_items = [] + self.data = data[1:] + self.dbid = data[0] + self.level = 1 + self.query_done = False + self.br_col = len(self.data) - 1 + + def getChildItem(self, row): + return self.child_items[row] + + def getParentItem(self): + return self.parent_item + + def getRow(self): + return self.row + + def Select(self): + self.query_done = True + + if not self.glb.have_disassembler: + return + + query = QSqlQuery(self.glb.db) + + QueryExec(query, "SELECT cpu, to_dso_id, to_symbol_id, to_sym_offset, short_name, long_name, build_id, sym_start, to_ip" + " FROM samples" + " INNER JOIN dsos ON samples.to_dso_id = dsos.id" + " INNER JOIN symbols ON samples.to_symbol_id = symbols.id" + " WHERE samples.id = " + str(self.dbid)) + if not query.next(): + return + cpu = query.value(0) + dso = query.value(1) + sym = query.value(2) + if dso == 0 or sym == 0: + return + off = query.value(3) + short_name = query.value(4) + long_name = query.value(5) + build_id = query.value(6) + sym_start = query.value(7) + ip = query.value(8) + + QueryExec(query, "SELECT samples.dso_id, symbol_id, sym_offset, sym_start" + " FROM samples" + " INNER JOIN symbols ON samples.symbol_id = symbols.id" + " WHERE samples.id > " + str(self.dbid) + " AND cpu = " + str(cpu) + + " ORDER BY samples.id" + " LIMIT 1") + if not query.next(): + return + if query.value(0) != dso: + # Cannot disassemble from one dso to another + return + bsym = query.value(1) + boff = query.value(2) + bsym_start = query.value(3) + if bsym == 0: + return + tot = bsym_start + boff + 1 - sym_start - off + if tot <= 0 or tot > 16384: + return + + inst = self.glb.disassembler.Instruction() + f = self.glb.FileFromNamesAndBuildId(short_name, long_name, build_id) + if not f: + return + mode = 0 if Is64Bit(f) else 1 + self.glb.disassembler.SetMode(inst, mode) + + buf_sz = tot + 16 + buf = create_string_buffer(tot + 16) + f.seek(sym_start + off) + buf.value = f.read(buf_sz) + buf_ptr = addressof(buf) + i = 0 + while tot > 0: + cnt, text = self.glb.disassembler.DisassembleOne(inst, buf_ptr, buf_sz, ip) + if cnt: + byte_str = tohex(ip).rjust(16) + for k in range(cnt): + byte_str += " %02x" % ord(buf[i]) + i += 1 + while k < 15: + byte_str += " " + k += 1 + self.child_items.append(BranchLevelTwoItem(0, self.br_col, byte_str + " " + text, self)) + self.child_count += 1 + else: + return + buf_ptr += cnt + tot -= cnt + buf_sz -= cnt + ip += cnt + + def childCount(self): + if not self.query_done: + self.Select() + if not self.child_count: + return -1 + return self.child_count + + def hasChildren(self): + if not self.query_done: + return True + return self.child_count > 0 + + def getData(self, column): + return self.data[column] + +# Brance data model root item + +class BranchRootItem(): + + def __init__(self): + self.child_count = 0 + self.child_items = [] + self.level = 0 + + def getChildItem(self, row): + return self.child_items[row] + + def getParentItem(self): + return None + + def getRow(self): + return 0 + + def childCount(self): + return self.child_count + + def hasChildren(self): + return self.child_count > 0 + + def getData(self, _column): + return "" + +# Calculate instructions per cycle + +def CalcIPC(cyc_cnt, insn_cnt): + if cyc_cnt and insn_cnt: + ipc = Decimal(float(insn_cnt) / cyc_cnt) + ipc = str(ipc.quantize(Decimal(".01"), rounding=ROUND_HALF_UP)) + else: + ipc = "0" + return ipc + +# Branch data preparation + +def BranchDataPrepBr(query, data): + data.append(tohex(query.value(8)).rjust(16) + " " + query.value(9) + offstr(query.value(10)) + + " (" + dsoname(query.value(11)) + ")" + " -> " + + tohex(query.value(12)) + " " + query.value(13) + offstr(query.value(14)) + + " (" + dsoname(query.value(15)) + ")") + +def BranchDataPrepIPC(query, data): + insn_cnt = query.value(16) + cyc_cnt = query.value(17) + ipc = CalcIPC(cyc_cnt, insn_cnt) + data.append(insn_cnt) + data.append(cyc_cnt) + data.append(ipc) + +def BranchDataPrep(query): + data = [] + for i in range(0, 8): + data.append(query.value(i)) + BranchDataPrepBr(query, data) + return data + +def BranchDataPrepWA(query): + data = [] + data.append(query.value(0)) + # Workaround pyside failing to handle large integers (i.e. time) in python3 by converting to a string + data.append("{:>19}".format(query.value(1))) + for i in range(2, 8): + data.append(query.value(i)) + BranchDataPrepBr(query, data) + return data + +def BranchDataWithIPCPrep(query): + data = [] + for i in range(0, 8): + data.append(query.value(i)) + BranchDataPrepIPC(query, data) + BranchDataPrepBr(query, data) + return data + +def BranchDataWithIPCPrepWA(query): + data = [] + data.append(query.value(0)) + # Workaround pyside failing to handle large integers (i.e. time) in python3 by converting to a string + data.append("{:>19}".format(query.value(1))) + for i in range(2, 8): + data.append(query.value(i)) + BranchDataPrepIPC(query, data) + BranchDataPrepBr(query, data) + return data + +# Branch data model + +class BranchModel(TreeModel): + + progress = Signal(object) + + def __init__(self, glb, event_id, where_clause, parent=None): + super(BranchModel, self).__init__(glb, None, parent) + self.event_id = event_id + self.more = True + self.populated = 0 + self.have_ipc = IsSelectable(glb.db, "samples", columns = "insn_count, cyc_count") + if self.have_ipc: + select_ipc = ", insn_count, cyc_count" + prep_fn = BranchDataWithIPCPrep + prep_wa_fn = BranchDataWithIPCPrepWA + else: + select_ipc = "" + prep_fn = BranchDataPrep + prep_wa_fn = BranchDataPrepWA + sql = ("SELECT samples.id, time, cpu, comm, pid, tid, branch_types.name," + " CASE WHEN in_tx = '0' THEN 'No' ELSE 'Yes' END," + " ip, symbols.name, sym_offset, dsos.short_name," + " to_ip, to_symbols.name, to_sym_offset, to_dsos.short_name" + + select_ipc + + " FROM samples" + " INNER JOIN comms ON comm_id = comms.id" + " INNER JOIN threads ON thread_id = threads.id" + " INNER JOIN branch_types ON branch_type = branch_types.id" + " INNER JOIN symbols ON symbol_id = symbols.id" + " INNER JOIN symbols to_symbols ON to_symbol_id = to_symbols.id" + " INNER JOIN dsos ON samples.dso_id = dsos.id" + " INNER JOIN dsos AS to_dsos ON samples.to_dso_id = to_dsos.id" + " WHERE samples.id > $$last_id$$" + where_clause + + " AND evsel_id = " + str(self.event_id) + + " ORDER BY samples.id" + " LIMIT " + str(glb_chunk_sz)) + if pyside_version_1 and sys.version_info[0] == 3: + prep = prep_fn + else: + prep = prep_wa_fn + self.fetcher = SQLFetcher(glb, sql, prep, self.AddSample) + self.fetcher.done.connect(self.Update) + self.fetcher.Fetch(glb_chunk_sz) + + def GetRoot(self): + return BranchRootItem() + + def columnCount(self, _parent=None): + if self.have_ipc: + return 11 + else: + return 8 + + def columnHeader(self, column): + if self.have_ipc: + return ("Time", "CPU", "Command", "PID", "TID", "Branch Type", "In Tx", "Insn Cnt", "Cyc Cnt", "IPC", "Branch")[column] + else: + return ("Time", "CPU", "Command", "PID", "TID", "Branch Type", "In Tx", "Branch")[column] + + def columnFont(self, column): + if self.have_ipc: + br_col = 10 + else: + br_col = 7 + if column != br_col: + return None + return QFont("Monospace") + + def DisplayData(self, item, index): + if item.level == 1: + self.FetchIfNeeded(item.row) + return item.getData(index.column()) + + def AddSample(self, data): + child = BranchLevelOneItem(self.glb, self.populated, data, self.root) + self.root.child_items.append(child) + self.populated += 1 + + def Update(self, fetched): + if not fetched: + self.more = False + self.progress.emit(0) + child_count = self.root.child_count + count = self.populated - child_count + if count > 0: + parent = QModelIndex() + self.beginInsertRows(parent, child_count, child_count + count - 1) + self.insertRows(child_count, count, parent) + self.root.child_count += count + self.endInsertRows() + self.progress.emit(self.root.child_count) + + def FetchMoreRecords(self, count): + current = self.root.child_count + if self.more: + self.fetcher.Fetch(count) + else: + self.progress.emit(0) + return current + + def HasMoreRecords(self): + return self.more + +# Report Variables + +class ReportVars(): + + def __init__(self, name = "", where_clause = "", limit = ""): + self.name = name + self.where_clause = where_clause + self.limit = limit + + def UniqueId(self): + return str(self.where_clause + ";" + self.limit) + +# Branch window + +class BranchWindow(QMdiSubWindow): + + def __init__(self, glb, event_id, report_vars, parent=None): + super(BranchWindow, self).__init__(parent) + + model_name = "Branch Events " + str(event_id) + " " + report_vars.UniqueId() + + self.model = LookupCreateModel(model_name, lambda: BranchModel(glb, event_id, report_vars.where_clause)) + + self.view = QTreeView() + self.view.setUniformRowHeights(True) + self.view.setSelectionMode(QAbstractItemView.ContiguousSelection) + self.view.CopyCellsToClipboard = CopyTreeCellsToClipboard + self.view.setModel(self.model) + + self.ResizeColumnsToContents() + + self.context_menu = TreeContextMenu(self.view) + + self.find_bar = FindBar(self, self, True) + + self.finder = ChildDataItemFinder(self.model.root) + + self.fetch_bar = FetchMoreRecordsBar(self.model, self) + + self.vbox = VBox(self.view, self.find_bar.Widget(), self.fetch_bar.Widget()) + + self.setWidget(self.vbox.Widget()) + + AddSubWindow(glb.mainwindow.mdi_area, self, report_vars.name + " Branch Events") + + def ResizeColumnToContents(self, column, n): + # Using the view's resizeColumnToContents() here is extrememly slow + # so implement a crude alternative + mm = "MM" if column else "MMMM" + font = self.view.font() + metrics = QFontMetrics(font) + max_w = 0 + for row in range(n): + val = self.model.root.child_items[row].data[column] + width = metrics.width(str(val) + mm) + max_w = width if width > max_w else max_w + val = self.model.columnHeader(column) + width = metrics.width(str(val) + mm) + max_w = width if width > max_w else max_w + self.view.setColumnWidth(column, max_w) + + def ResizeColumnsToContents(self): + n = min(self.model.root.child_count, 100) + if n < 1: + # No data yet, so connect a signal to notify when there is + self.model.rowsInserted.connect(self.UpdateColumnWidths) + return + columns = self.model.columnCount() + for i in range(columns): + self.ResizeColumnToContents(i, n) + + def UpdateColumnWidths(self, *_x): + # This only needs to be done once, so disconnect the signal now + self.model.rowsInserted.disconnect(self.UpdateColumnWidths) + self.ResizeColumnsToContents() + + def Find(self, value, direction, pattern, context): + self.view.setFocus() + self.find_bar.Busy() + self.finder.Find(value, direction, pattern, context, self.FindDone) + + def FindDone(self, row): + self.find_bar.Idle() + if row >= 0: + self.view.setCurrentIndex(self.model.index(row, 0, QModelIndex())) + else: + self.find_bar.NotFound() + +# Line edit data item + +class LineEditDataItem(object): + + def __init__(self, glb, label, placeholder_text, parent, item_id = "", default = ""): + self.glb = glb + self.label = label + self.placeholder_text = placeholder_text + self.parent = parent + self.id = item_id + + self.value = default + + self.widget = QLineEdit(default) + self.widget.editingFinished.connect(self.Validate) + self.widget.textChanged.connect(self.Invalidate) + self.red = False + self.error = "" + self.validated = True + + if placeholder_text: + self.widget.setPlaceholderText(placeholder_text) + + def TurnTextRed(self): + if not self.red: + palette = QPalette() + palette.setColor(QPalette.Text,Qt.red) + self.widget.setPalette(palette) + self.red = True + + def TurnTextNormal(self): + if self.red: + palette = QPalette() + self.widget.setPalette(palette) + self.red = False + + def InvalidValue(self, value): + self.value = "" + self.TurnTextRed() + self.error = self.label + " invalid value '" + value + "'" + self.parent.ShowMessage(self.error) + + def Invalidate(self): + self.validated = False + + def DoValidate(self, input_string): + self.value = input_string.strip() + + def Validate(self): + self.validated = True + self.error = "" + self.TurnTextNormal() + self.parent.ClearMessage() + input_string = self.widget.text() + if not len(input_string.strip()): + self.value = "" + return + self.DoValidate(input_string) + + def IsValid(self): + if not self.validated: + self.Validate() + if len(self.error): + self.parent.ShowMessage(self.error) + return False + return True + + def IsNumber(self, value): + try: + x = int(value) + except ValueError: + x = 0 + return str(x) == value + +# Non-negative integer ranges dialog data item + +class NonNegativeIntegerRangesDataItem(LineEditDataItem): + + def __init__(self, glb, label, placeholder_text, column_name, parent): + super(NonNegativeIntegerRangesDataItem, self).__init__(glb, label, placeholder_text, parent) + + self.column_name = column_name + + def DoValidate(self, input_string): + singles = [] + ranges = [] + for value in [x.strip() for x in input_string.split(",")]: + if "-" in value: + vrange = value.split("-") + if len(vrange) != 2 or not self.IsNumber(vrange[0]) or not self.IsNumber(vrange[1]): + return self.InvalidValue(value) + ranges.append(vrange) + else: + if not self.IsNumber(value): + return self.InvalidValue(value) + singles.append(value) + ranges = [("(" + self.column_name + " >= " + r[0] + " AND " + self.column_name + " <= " + r[1] + ")") for r in ranges] + if len(singles): + ranges.append(self.column_name + " IN (" + ",".join(singles) + ")") + self.value = " OR ".join(ranges) + +# Positive integer dialog data item + +class PositiveIntegerDataItem(LineEditDataItem): + + def DoValidate(self, input_string): + if not self.IsNumber(input_string.strip()): + return self.InvalidValue(input_string) + value = int(input_string.strip()) + if value <= 0: + return self.InvalidValue(input_string) + self.value = str(value) + +# Dialog data item converted and validated using a SQL table + +class SQLTableDataItem(LineEditDataItem): + + def __init__(self, glb, label, placeholder_text, table_name, match_column, column_name1, column_name2, parent): + super(SQLTableDataItem, self).__init__(glb, label, placeholder_text, parent) + + self.table_name = table_name + self.match_column = match_column + self.column_name1 = column_name1 + self.column_name2 = column_name2 + + def ValueToIds(self, value): + ids = [] + query = QSqlQuery(self.glb.db) + stmt = "SELECT id FROM " + self.table_name + " WHERE " + self.match_column + " = '" + value + "'" + ret = query.exec_(stmt) + if ret: + while query.next(): + ids.append(str(query.value(0))) + return ids + + def DoValidate(self, input_string): + all_ids = [] + for value in [x.strip() for x in input_string.split(",")]: + ids = self.ValueToIds(value) + if len(ids): + all_ids.extend(ids) + else: + return self.InvalidValue(value) + self.value = self.column_name1 + " IN (" + ",".join(all_ids) + ")" + if self.column_name2: + self.value = "( " + self.value + " OR " + self.column_name2 + " IN (" + ",".join(all_ids) + ") )" + +# Sample time ranges dialog data item converted and validated using 'samples' SQL table + +class SampleTimeRangesDataItem(LineEditDataItem): + + def __init__(self, glb, label, placeholder_text, column_name, parent): + self.column_name = column_name + + self.last_id = 0 + self.first_time = 0 + self.last_time = 2 ** 64 + + query = QSqlQuery(glb.db) + QueryExec(query, "SELECT id, time FROM samples ORDER BY id DESC LIMIT 1") + if query.next(): + self.last_id = int(query.value(0)) + self.first_time = int(glb.HostStartTime()) + self.last_time = int(glb.HostFinishTime()) + if placeholder_text: + placeholder_text += ", between " + str(self.first_time) + " and " + str(self.last_time) + + super(SampleTimeRangesDataItem, self).__init__(glb, label, placeholder_text, parent) + + def IdBetween(self, query, lower_id, higher_id, order): + QueryExec(query, "SELECT id FROM samples WHERE id > " + str(lower_id) + " AND id < " + str(higher_id) + " ORDER BY id " + order + " LIMIT 1") + if query.next(): + return True, int(query.value(0)) + else: + return False, 0 + + def BinarySearchTime(self, lower_id, higher_id, target_time, get_floor): + query = QSqlQuery(self.glb.db) + while True: + next_id = int((lower_id + higher_id) / 2) + QueryExec(query, "SELECT time FROM samples WHERE id = " + str(next_id)) + if not query.next(): + ok, dbid = self.IdBetween(query, lower_id, next_id, "DESC") + if not ok: + ok, dbid = self.IdBetween(query, next_id, higher_id, "") + if not ok: + return str(higher_id) + next_id = dbid + QueryExec(query, "SELECT time FROM samples WHERE id = " + str(next_id)) + next_time = int(query.value(0)) + if get_floor: + if target_time > next_time: + lower_id = next_id + else: + higher_id = next_id + if higher_id <= lower_id + 1: + return str(higher_id) + else: + if target_time >= next_time: + lower_id = next_id + else: + higher_id = next_id + if higher_id <= lower_id + 1: + return str(lower_id) + + def ConvertRelativeTime(self, val): + mult = 1 + suffix = val[-2:] + if suffix == "ms": + mult = 1000000 + elif suffix == "us": + mult = 1000 + elif suffix == "ns": + mult = 1 + else: + return val + val = val[:-2].strip() + if not self.IsNumber(val): + return val + val = int(val) * mult + if val >= 0: + val += self.first_time + else: + val += self.last_time + return str(val) + + def ConvertTimeRange(self, vrange): + if vrange[0] == "": + vrange[0] = str(self.first_time) + if vrange[1] == "": + vrange[1] = str(self.last_time) + vrange[0] = self.ConvertRelativeTime(vrange[0]) + vrange[1] = self.ConvertRelativeTime(vrange[1]) + if not self.IsNumber(vrange[0]) or not self.IsNumber(vrange[1]): + return False + beg_range = max(int(vrange[0]), self.first_time) + end_range = min(int(vrange[1]), self.last_time) + if beg_range > self.last_time or end_range < self.first_time: + return False + vrange[0] = self.BinarySearchTime(0, self.last_id, beg_range, True) + vrange[1] = self.BinarySearchTime(1, self.last_id + 1, end_range, False) + return True + + def AddTimeRange(self, value, ranges): + n = value.count("-") + if n == 1: + pass + elif n == 2: + if value.split("-")[1].strip() == "": + n = 1 + elif n == 3: + n = 2 + else: + return False + pos = findnth(value, "-", n) + vrange = [value[:pos].strip() ,value[pos+1:].strip()] + if self.ConvertTimeRange(vrange): + ranges.append(vrange) + return True + return False + + def DoValidate(self, input_string): + ranges = [] + for value in [x.strip() for x in input_string.split(",")]: + if not self.AddTimeRange(value, ranges): + return self.InvalidValue(value) + ranges = [("(" + self.column_name + " >= " + r[0] + " AND " + self.column_name + " <= " + r[1] + ")") for r in ranges] + self.value = " OR ".join(ranges) + +# Report Dialog Base + +class ReportDialogBase(QDialog): + + def __init__(self, glb, title, items, partial, parent=None): + super(ReportDialogBase, self).__init__(parent) + + self.glb = glb + + self.report_vars = ReportVars() + + self.setWindowTitle(title) + self.setMinimumWidth(600) + + self.data_items = [x(glb, self) for x in items] + + self.partial = partial + + self.grid = QGridLayout() + + for row in range(len(self.data_items)): + self.grid.addWidget(QLabel(self.data_items[row].label), row, 0) + self.grid.addWidget(self.data_items[row].widget, row, 1) + + self.status = QLabel() + + self.ok_button = QPushButton("Ok", self) + self.ok_button.setDefault(True) + self.ok_button.released.connect(self.Ok) + self.ok_button.setSizePolicy(QSizePolicy.Fixed, QSizePolicy.Fixed) + + self.cancel_button = QPushButton("Cancel", self) + self.cancel_button.released.connect(self.reject) + self.cancel_button.setSizePolicy(QSizePolicy.Fixed, QSizePolicy.Fixed) + + self.hbox = QHBoxLayout() + #self.hbox.addStretch() + self.hbox.addWidget(self.status) + self.hbox.addWidget(self.ok_button) + self.hbox.addWidget(self.cancel_button) + + self.vbox = QVBoxLayout() + self.vbox.addLayout(self.grid) + self.vbox.addLayout(self.hbox) + + self.setLayout(self.vbox) + + def Ok(self): + rep_vars = self.report_vars + for d in self.data_items: + if d.id == "REPORTNAME": + rep_vars.name = d.value + if not rep_vars.name: + self.ShowMessage("Report name is required") + return + for d in self.data_items: + if not d.IsValid(): + return + for d in self.data_items[1:]: + if d.id == "LIMIT": + rep_vars.limit = d.value + elif len(d.value): + if len(rep_vars.where_clause): + rep_vars.where_clause += " AND " + rep_vars.where_clause += d.value + if len(rep_vars.where_clause): + if self.partial: + rep_vars.where_clause = " AND ( " + rep_vars.where_clause + " ) " + else: + rep_vars.where_clause = " WHERE " + rep_vars.where_clause + " " + self.accept() + + def ShowMessage(self, msg): + self.status.setText("<font color=#FF0000>" + msg) + + def ClearMessage(self): + self.status.setText("") + +# Selected branch report creation dialog + +class SelectedBranchDialog(ReportDialogBase): + + def __init__(self, glb, parent=None): + title = "Selected Branches" + items = (lambda g, p: LineEditDataItem(g, "Report name:", "Enter a name to appear in the window title bar", p, "REPORTNAME"), + lambda g, p: SampleTimeRangesDataItem(g, "Time ranges:", "Enter time ranges", "samples.id", p), + lambda g, p: NonNegativeIntegerRangesDataItem(g, "CPUs:", "Enter CPUs or ranges e.g. 0,5-6", "cpu", p), + lambda g, p: SQLTableDataItem(g, "Commands:", "Only branches with these commands will be included", "comms", "comm", "comm_id", "", p), + lambda g, p: SQLTableDataItem(g, "PIDs:", "Only branches with these process IDs will be included", "threads", "pid", "thread_id", "", p), + lambda g, p: SQLTableDataItem(g, "TIDs:", "Only branches with these thread IDs will be included", "threads", "tid", "thread_id", "", p), + lambda g, p: SQLTableDataItem(g, "DSOs:", "Only branches with these DSOs will be included", "dsos", "short_name", "samples.dso_id", "to_dso_id", p), + lambda g, p: SQLTableDataItem(g, "Symbols:", "Only branches with these symbols will be included", "symbols", "name", "symbol_id", "to_symbol_id", p), + lambda g, p: LineEditDataItem(g, "Raw SQL clause: ", "Enter a raw SQL WHERE clause", p)) + super(SelectedBranchDialog, self).__init__(glb, title, items, True, parent) + +# Event list + +def GetEventList(db): + events = [] + query = QSqlQuery(db) + QueryExec(query, "SELECT name FROM selected_events WHERE id > 0 ORDER BY id") + while query.next(): + events.append(query.value(0)) + return events + +# Is a table selectable + +def IsSelectable(db, table, sql = "", columns = "*"): + query = QSqlQuery(db) + try: + QueryExec(query, "SELECT " + columns + " FROM " + table + " " + sql + " LIMIT 1") + except RuntimeError: + return False + return True + +# SQL table data model item + +class SQLTableItem(): + + def __init__(self, row, data): + self.row = row + self.data = data + + def getData(self, column): + return self.data[column] + +# SQL table data model + +class SQLTableModel(TableModel): + + progress = Signal(object) + + def __init__(self, glb, sql, column_headers, parent=None): + super(SQLTableModel, self).__init__(parent) + self.glb = glb + self.more = True + self.populated = 0 + self.column_headers = column_headers + self.fetcher = SQLFetcher(glb, sql, lambda x, y=len(column_headers): self.SQLTableDataPrep(x, y), self.AddSample) + self.fetcher.done.connect(self.Update) + self.fetcher.Fetch(glb_chunk_sz) + + def DisplayData(self, item, index): + self.FetchIfNeeded(item.row) + return item.getData(index.column()) + + def AddSample(self, data): + child = SQLTableItem(self.populated, data) + self.child_items.append(child) + self.populated += 1 + + def Update(self, fetched): + if not fetched: + self.more = False + self.progress.emit(0) + child_count = self.child_count + count = self.populated - child_count + if count > 0: + parent = QModelIndex() + self.beginInsertRows(parent, child_count, child_count + count - 1) + self.insertRows(child_count, count, parent) + self.child_count += count + self.endInsertRows() + self.progress.emit(self.child_count) + + def FetchMoreRecords(self, count): + current = self.child_count + if self.more: + self.fetcher.Fetch(count) + else: + self.progress.emit(0) + return current + + def HasMoreRecords(self): + return self.more + + def columnCount(self, _parent=None): + return len(self.column_headers) + + def columnHeader(self, column): + return self.column_headers[column] + + def SQLTableDataPrep(self, query, count): + data = [] + for i in range(count): + data.append(query.value(i)) + return data + +# SQL automatic table data model + +class SQLAutoTableModel(SQLTableModel): + + def __init__(self, glb, table_name, parent=None): + sql = "SELECT * FROM " + table_name + " WHERE id > $$last_id$$ ORDER BY id LIMIT " + str(glb_chunk_sz) + if table_name == "comm_threads_view": + # For now, comm_threads_view has no id column + sql = "SELECT * FROM " + table_name + " WHERE comm_id > $$last_id$$ ORDER BY comm_id LIMIT " + str(glb_chunk_sz) + column_headers = [] + query = QSqlQuery(glb.db) + if glb.dbref.is_sqlite3: + QueryExec(query, "PRAGMA table_info(" + table_name + ")") + while query.next(): + column_headers.append(query.value(1)) + if table_name == "sqlite_master": + sql = "SELECT * FROM " + table_name + else: + if table_name[:19] == "information_schema.": + sql = "SELECT * FROM " + table_name + select_table_name = table_name[19:] + schema = "information_schema" + else: + select_table_name = table_name + schema = "public" + QueryExec(query, "SELECT column_name FROM information_schema.columns WHERE table_schema = '" + schema + "' and table_name = '" + select_table_name + "'") + while query.next(): + column_headers.append(query.value(0)) + if pyside_version_1 and sys.version_info[0] == 3: + if table_name == "samples_view": + self.SQLTableDataPrep = self.samples_view_DataPrep + if table_name == "samples": + self.SQLTableDataPrep = self.samples_DataPrep + super(SQLAutoTableModel, self).__init__(glb, sql, column_headers, parent) + + def samples_view_DataPrep(self, query, count): + data = [] + data.append(query.value(0)) + # Workaround pyside failing to handle large integers (i.e. time) in python3 by converting to a string + data.append("{:>19}".format(query.value(1))) + for i in range(2, count): + data.append(query.value(i)) + return data + + def samples_DataPrep(self, query, count): + data = [] + for i in range(9): + data.append(query.value(i)) + # Workaround pyside failing to handle large integers (i.e. time) in python3 by converting to a string + data.append("{:>19}".format(query.value(9))) + for i in range(10, count): + data.append(query.value(i)) + return data + +# Base class for custom ResizeColumnsToContents + +class ResizeColumnsToContentsBase(QObject): + + def __init__(self, parent=None): + super(ResizeColumnsToContentsBase, self).__init__(parent) + + def ResizeColumnToContents(self, column, n): + # Using the view's resizeColumnToContents() here is extrememly slow + # so implement a crude alternative + font = self.view.font() + metrics = QFontMetrics(font) + max_w = 0 + for row in range(n): + val = self.data_model.child_items[row].data[column] + width = metrics.width(str(val) + "MM") + max_w = width if width > max_w else max_w + val = self.data_model.columnHeader(column) + width = metrics.width(str(val) + "MM") + max_w = width if width > max_w else max_w + self.view.setColumnWidth(column, max_w) + + def ResizeColumnsToContents(self): + n = min(self.data_model.child_count, 100) + if n < 1: + # No data yet, so connect a signal to notify when there is + self.data_model.rowsInserted.connect(self.UpdateColumnWidths) + return + columns = self.data_model.columnCount() + for i in range(columns): + self.ResizeColumnToContents(i, n) + + def UpdateColumnWidths(self, *_x): + # This only needs to be done once, so disconnect the signal now + self.data_model.rowsInserted.disconnect(self.UpdateColumnWidths) + self.ResizeColumnsToContents() + +# Convert value to CSV + +def ToCSValue(val): + if '"' in val: + val = val.replace('"', '""') + if "," in val or '"' in val: + val = '"' + val + '"' + return val + +# Key to sort table model indexes by row / column, assuming fewer than 1000 columns + +glb_max_cols = 1000 + +def RowColumnKey(a): + return a.row() * glb_max_cols + a.column() + +# Copy selected table cells to clipboard + +def CopyTableCellsToClipboard(view, as_csv=False, with_hdr=False): + indexes = sorted(view.selectedIndexes(), key=RowColumnKey) + idx_cnt = len(indexes) + if not idx_cnt: + return + if idx_cnt == 1: + with_hdr=False + min_row = indexes[0].row() + max_row = indexes[0].row() + min_col = indexes[0].column() + max_col = indexes[0].column() + for i in indexes: + min_row = min(min_row, i.row()) + max_row = max(max_row, i.row()) + min_col = min(min_col, i.column()) + max_col = max(max_col, i.column()) + if max_col > glb_max_cols: + raise RuntimeError("glb_max_cols is too low") + max_width = [0] * (1 + max_col - min_col) + for i in indexes: + c = i.column() - min_col + max_width[c] = max(max_width[c], len(str(i.data()))) + text = "" + pad = "" + sep = "" + if with_hdr: + model = indexes[0].model() + for col in range(min_col, max_col + 1): + val = model.headerData(col, Qt.Horizontal, Qt.DisplayRole) + if as_csv: + text += sep + ToCSValue(val) + sep = "," + else: + c = col - min_col + max_width[c] = max(max_width[c], len(val)) + width = max_width[c] + align = model.headerData(col, Qt.Horizontal, Qt.TextAlignmentRole) + if align & Qt.AlignRight: + val = val.rjust(width) + text += pad + sep + val + pad = " " * (width - len(val)) + sep = " " + text += "\n" + pad = "" + sep = "" + last_row = min_row + for i in indexes: + if i.row() > last_row: + last_row = i.row() + text += "\n" + pad = "" + sep = "" + if as_csv: + text += sep + ToCSValue(str(i.data())) + sep = "," + else: + width = max_width[i.column() - min_col] + if i.data(Qt.TextAlignmentRole) & Qt.AlignRight: + val = str(i.data()).rjust(width) + else: + val = str(i.data()) + text += pad + sep + val + pad = " " * (width - len(val)) + sep = " " + QApplication.clipboard().setText(text) + +def CopyTreeCellsToClipboard(view, as_csv=False, with_hdr=False): + indexes = view.selectedIndexes() + if not len(indexes): + return + + selection = view.selectionModel() + + first = None + for i in indexes: + above = view.indexAbove(i) + if not selection.isSelected(above): + first = i + break + + if first is None: + raise RuntimeError("CopyTreeCellsToClipboard internal error") + + model = first.model() + row_cnt = 0 + col_cnt = model.columnCount(first) + max_width = [0] * col_cnt + + indent_sz = 2 + indent_str = " " * indent_sz + + expanded_mark_sz = 2 + if sys.version_info[0] == 3: + expanded_mark = "\u25BC " + not_expanded_mark = "\u25B6 " + else: + expanded_mark = chr(0x25BC) + " " + not_expanded_mark = chr(0x25B6) + " " + leaf_mark = " " + + if not as_csv: + pos = first + while True: + row_cnt += 1 + row = pos.row() + for c in range(col_cnt): + i = pos.sibling(row, c) + if c: + n = len(str(i.data())) + else: + n = len(str(i.data()).strip()) + n += (i.internalPointer().level - 1) * indent_sz + n += expanded_mark_sz + max_width[c] = max(max_width[c], n) + pos = view.indexBelow(pos) + if not selection.isSelected(pos): + break + + text = "" + pad = "" + sep = "" + if with_hdr: + for c in range(col_cnt): + val = model.headerData(c, Qt.Horizontal, Qt.DisplayRole).strip() + if as_csv: + text += sep + ToCSValue(val) + sep = "," + else: + max_width[c] = max(max_width[c], len(val)) + width = max_width[c] + align = model.headerData(c, Qt.Horizontal, Qt.TextAlignmentRole) + if align & Qt.AlignRight: + val = val.rjust(width) + text += pad + sep + val + pad = " " * (width - len(val)) + sep = " " + text += "\n" + pad = "" + sep = "" + + pos = first + while True: + row = pos.row() + for c in range(col_cnt): + i = pos.sibling(row, c) + val = str(i.data()) + if not c: + if model.hasChildren(i): + if view.isExpanded(i): + mark = expanded_mark + else: + mark = not_expanded_mark + else: + mark = leaf_mark + val = indent_str * (i.internalPointer().level - 1) + mark + val.strip() + if as_csv: + text += sep + ToCSValue(val) + sep = "," + else: + width = max_width[c] + if c and i.data(Qt.TextAlignmentRole) & Qt.AlignRight: + val = val.rjust(width) + text += pad + sep + val + pad = " " * (width - len(val)) + sep = " " + pos = view.indexBelow(pos) + if not selection.isSelected(pos): + break + text = text.rstrip() + "\n" + pad = "" + sep = "" + + QApplication.clipboard().setText(text) + +def CopyCellsToClipboard(view, as_csv=False, with_hdr=False): + view.CopyCellsToClipboard(view, as_csv, with_hdr) + +def CopyCellsToClipboardHdr(view): + CopyCellsToClipboard(view, False, True) + +def CopyCellsToClipboardCSV(view): + CopyCellsToClipboard(view, True, True) + +# Context menu + +class ContextMenu(object): + + def __init__(self, view): + self.view = view + self.view.setContextMenuPolicy(Qt.CustomContextMenu) + self.view.customContextMenuRequested.connect(self.ShowContextMenu) + + def ShowContextMenu(self, pos): + menu = QMenu(self.view) + self.AddActions(menu) + menu.exec_(self.view.mapToGlobal(pos)) + + def AddCopy(self, menu): + menu.addAction(CreateAction("&Copy selection", "Copy to clipboard", lambda: CopyCellsToClipboardHdr(self.view), self.view)) + menu.addAction(CreateAction("Copy selection as CS&V", "Copy to clipboard as CSV", lambda: CopyCellsToClipboardCSV(self.view), self.view)) + + def AddActions(self, menu): + self.AddCopy(menu) + +class TreeContextMenu(ContextMenu): + + def AddActions(self, menu): + i = self.view.currentIndex() + text = str(i.data()).strip() + if len(text): + menu.addAction(CreateAction('Copy "' + text + '"', "Copy to clipboard", lambda: QApplication.clipboard().setText(text), self.view)) + self.AddCopy(menu) + +# Table window + +class TableWindow(QMdiSubWindow, ResizeColumnsToContentsBase): + + def __init__(self, glb, table_name, parent=None): + super(TableWindow, self).__init__(parent) + + self.data_model = LookupCreateModel(table_name + " Table", lambda: SQLAutoTableModel(glb, table_name)) + + self.model = QSortFilterProxyModel() + self.model.setSourceModel(self.data_model) + + self.view = QTableView() + self.view.setModel(self.model) + self.view.setEditTriggers(QAbstractItemView.NoEditTriggers) + self.view.verticalHeader().setVisible(False) + self.view.sortByColumn(-1, Qt.AscendingOrder) + self.view.setSortingEnabled(True) + self.view.setSelectionMode(QAbstractItemView.ContiguousSelection) + self.view.CopyCellsToClipboard = CopyTableCellsToClipboard + + self.ResizeColumnsToContents() + + self.context_menu = ContextMenu(self.view) + + self.find_bar = FindBar(self, self, True) + + self.finder = ChildDataItemFinder(self.data_model) + + self.fetch_bar = FetchMoreRecordsBar(self.data_model, self) + + self.vbox = VBox(self.view, self.find_bar.Widget(), self.fetch_bar.Widget()) + + self.setWidget(self.vbox.Widget()) + + AddSubWindow(glb.mainwindow.mdi_area, self, table_name + " Table") + + def Find(self, value, direction, pattern, context): + self.view.setFocus() + self.find_bar.Busy() + self.finder.Find(value, direction, pattern, context, self.FindDone) + + def FindDone(self, row): + self.find_bar.Idle() + if row >= 0: + self.view.setCurrentIndex(self.model.mapFromSource(self.data_model.index(row, 0, QModelIndex()))) + else: + self.find_bar.NotFound() + +# Table list + +def GetTableList(glb): + tables = [] + query = QSqlQuery(glb.db) + if glb.dbref.is_sqlite3: + QueryExec(query, "SELECT name FROM sqlite_master WHERE type IN ( 'table' , 'view' ) ORDER BY name") + else: + QueryExec(query, "SELECT table_name FROM information_schema.tables WHERE table_schema = 'public' AND table_type IN ( 'BASE TABLE' , 'VIEW' ) ORDER BY table_name") + while query.next(): + tables.append(query.value(0)) + if glb.dbref.is_sqlite3: + tables.append("sqlite_master") + else: + tables.append("information_schema.tables") + tables.append("information_schema.views") + tables.append("information_schema.columns") + return tables + +# Top Calls data model + +class TopCallsModel(SQLTableModel): + + def __init__(self, glb, report_vars, parent=None): + text = "" + if not glb.dbref.is_sqlite3: + text = "::text" + limit = "" + if len(report_vars.limit): + limit = " LIMIT " + report_vars.limit + sql = ("SELECT comm, pid, tid, name," + " CASE" + " WHEN (short_name = '[kernel.kallsyms]') THEN '[kernel]'" + text + + " ELSE short_name" + " END AS dso," + " call_time, return_time, (return_time - call_time) AS elapsed_time, branch_count, " + " CASE" + " WHEN (calls.flags = 1) THEN 'no call'" + text + + " WHEN (calls.flags = 2) THEN 'no return'" + text + + " WHEN (calls.flags = 3) THEN 'no call/return'" + text + + " ELSE ''" + text + + " END AS flags" + " FROM calls" + " INNER JOIN call_paths ON calls.call_path_id = call_paths.id" + " INNER JOIN symbols ON call_paths.symbol_id = symbols.id" + " INNER JOIN dsos ON symbols.dso_id = dsos.id" + " INNER JOIN comms ON calls.comm_id = comms.id" + " INNER JOIN threads ON calls.thread_id = threads.id" + + report_vars.where_clause + + " ORDER BY elapsed_time DESC" + + limit + ) + column_headers = ("Command", "PID", "TID", "Symbol", "Object", "Call Time", "Return Time", "Elapsed Time (ns)", "Branch Count", "Flags") + self.alignment = (Qt.AlignLeft, Qt.AlignLeft, Qt.AlignLeft, Qt.AlignLeft, Qt.AlignLeft, Qt.AlignLeft, Qt.AlignLeft, Qt.AlignRight, Qt.AlignRight, Qt.AlignLeft) + super(TopCallsModel, self).__init__(glb, sql, column_headers, parent) + + def columnAlignment(self, column): + return self.alignment[column] + +# Top Calls report creation dialog + +class TopCallsDialog(ReportDialogBase): + + def __init__(self, glb, parent=None): + title = "Top Calls by Elapsed Time" + items = (lambda g, p: LineEditDataItem(g, "Report name:", "Enter a name to appear in the window title bar", p, "REPORTNAME"), + lambda g, p: SQLTableDataItem(g, "Commands:", "Only calls with these commands will be included", "comms", "comm", "comm_id", "", p), + lambda g, p: SQLTableDataItem(g, "PIDs:", "Only calls with these process IDs will be included", "threads", "pid", "thread_id", "", p), + lambda g, p: SQLTableDataItem(g, "TIDs:", "Only calls with these thread IDs will be included", "threads", "tid", "thread_id", "", p), + lambda g, p: SQLTableDataItem(g, "DSOs:", "Only calls with these DSOs will be included", "dsos", "short_name", "dso_id", "", p), + lambda g, p: SQLTableDataItem(g, "Symbols:", "Only calls with these symbols will be included", "symbols", "name", "symbol_id", "", p), + lambda g, p: LineEditDataItem(g, "Raw SQL clause: ", "Enter a raw SQL WHERE clause", p), + lambda g, p: PositiveIntegerDataItem(g, "Record limit:", "Limit selection to this number of records", p, "LIMIT", "100")) + super(TopCallsDialog, self).__init__(glb, title, items, False, parent) + +# Top Calls window + +class TopCallsWindow(QMdiSubWindow, ResizeColumnsToContentsBase): + + def __init__(self, glb, report_vars, parent=None): + super(TopCallsWindow, self).__init__(parent) + + self.data_model = LookupCreateModel("Top Calls " + report_vars.UniqueId(), lambda: TopCallsModel(glb, report_vars)) + self.model = self.data_model + + self.view = QTableView() + self.view.setModel(self.model) + self.view.setEditTriggers(QAbstractItemView.NoEditTriggers) + self.view.verticalHeader().setVisible(False) + self.view.setSelectionMode(QAbstractItemView.ContiguousSelection) + self.view.CopyCellsToClipboard = CopyTableCellsToClipboard + + self.context_menu = ContextMenu(self.view) + + self.ResizeColumnsToContents() + + self.find_bar = FindBar(self, self, True) + + self.finder = ChildDataItemFinder(self.model) + + self.fetch_bar = FetchMoreRecordsBar(self.data_model, self) + + self.vbox = VBox(self.view, self.find_bar.Widget(), self.fetch_bar.Widget()) + + self.setWidget(self.vbox.Widget()) + + AddSubWindow(glb.mainwindow.mdi_area, self, report_vars.name) + + def Find(self, value, direction, pattern, context): + self.view.setFocus() + self.find_bar.Busy() + self.finder.Find(value, direction, pattern, context, self.FindDone) + + def FindDone(self, row): + self.find_bar.Idle() + if row >= 0: + self.view.setCurrentIndex(self.model.index(row, 0, QModelIndex())) + else: + self.find_bar.NotFound() + +# Action Definition + +def CreateAction(label, tip, callback, parent=None, shortcut=None): + action = QAction(label, parent) + if shortcut != None: + action.setShortcuts(shortcut) + action.setStatusTip(tip) + action.triggered.connect(callback) + return action + +# Typical application actions + +def CreateExitAction(app, parent=None): + return CreateAction("&Quit", "Exit the application", app.closeAllWindows, parent, QKeySequence.Quit) + +# Typical MDI actions + +def CreateCloseActiveWindowAction(mdi_area): + return CreateAction("Cl&ose", "Close the active window", mdi_area.closeActiveSubWindow, mdi_area) + +def CreateCloseAllWindowsAction(mdi_area): + return CreateAction("Close &All", "Close all the windows", mdi_area.closeAllSubWindows, mdi_area) + +def CreateTileWindowsAction(mdi_area): + return CreateAction("&Tile", "Tile the windows", mdi_area.tileSubWindows, mdi_area) + +def CreateCascadeWindowsAction(mdi_area): + return CreateAction("&Cascade", "Cascade the windows", mdi_area.cascadeSubWindows, mdi_area) + +def CreateNextWindowAction(mdi_area): + return CreateAction("Ne&xt", "Move the focus to the next window", mdi_area.activateNextSubWindow, mdi_area, QKeySequence.NextChild) + +def CreatePreviousWindowAction(mdi_area): + return CreateAction("Pre&vious", "Move the focus to the previous window", mdi_area.activatePreviousSubWindow, mdi_area, QKeySequence.PreviousChild) + +# Typical MDI window menu + +class WindowMenu(): + + def __init__(self, mdi_area, menu): + self.mdi_area = mdi_area + self.window_menu = menu.addMenu("&Windows") + self.close_active_window = CreateCloseActiveWindowAction(mdi_area) + self.close_all_windows = CreateCloseAllWindowsAction(mdi_area) + self.tile_windows = CreateTileWindowsAction(mdi_area) + self.cascade_windows = CreateCascadeWindowsAction(mdi_area) + self.next_window = CreateNextWindowAction(mdi_area) + self.previous_window = CreatePreviousWindowAction(mdi_area) + self.window_menu.aboutToShow.connect(self.Update) + + def Update(self): + self.window_menu.clear() + sub_window_count = len(self.mdi_area.subWindowList()) + have_sub_windows = sub_window_count != 0 + self.close_active_window.setEnabled(have_sub_windows) + self.close_all_windows.setEnabled(have_sub_windows) + self.tile_windows.setEnabled(have_sub_windows) + self.cascade_windows.setEnabled(have_sub_windows) + self.next_window.setEnabled(have_sub_windows) + self.previous_window.setEnabled(have_sub_windows) + self.window_menu.addAction(self.close_active_window) + self.window_menu.addAction(self.close_all_windows) + self.window_menu.addSeparator() + self.window_menu.addAction(self.tile_windows) + self.window_menu.addAction(self.cascade_windows) + self.window_menu.addSeparator() + self.window_menu.addAction(self.next_window) + self.window_menu.addAction(self.previous_window) + if sub_window_count == 0: + return + self.window_menu.addSeparator() + nr = 1 + for sub_window in self.mdi_area.subWindowList(): + label = str(nr) + " " + sub_window.name + if nr < 10: + label = "&" + label + action = self.window_menu.addAction(label) + action.setCheckable(True) + action.setChecked(sub_window == self.mdi_area.activeSubWindow()) + action.triggered.connect(lambda a=None,x=nr: self.setActiveSubWindow(x)) + self.window_menu.addAction(action) + nr += 1 + + def setActiveSubWindow(self, nr): + self.mdi_area.setActiveSubWindow(self.mdi_area.subWindowList()[nr - 1]) + +# Help text + +glb_help_text = """ +<h1>Contents</h1> +<style> +p.c1 { + text-indent: 40px; +} +p.c2 { + text-indent: 80px; +} +} +</style> +<p class=c1><a href=#reports>1. Reports</a></p> +<p class=c2><a href=#callgraph>1.1 Context-Sensitive Call Graph</a></p> +<p class=c2><a href=#calltree>1.2 Call Tree</a></p> +<p class=c2><a href=#allbranches>1.3 All branches</a></p> +<p class=c2><a href=#selectedbranches>1.4 Selected branches</a></p> +<p class=c2><a href=#topcallsbyelapsedtime>1.5 Top calls by elapsed time</a></p> +<p class=c1><a href=#charts>2. Charts</a></p> +<p class=c2><a href=#timechartbycpu>2.1 Time chart by CPU</a></p> +<p class=c1><a href=#tables>3. Tables</a></p> +<h1 id=reports>1. Reports</h1> +<h2 id=callgraph>1.1 Context-Sensitive Call Graph</h2> +The result is a GUI window with a tree representing a context-sensitive +call-graph. Expanding a couple of levels of the tree and adjusting column +widths to suit will display something like: +<pre> + Call Graph: pt_example +Call Path Object Count Time(ns) Time(%) Branch Count Branch Count(%) +v- ls + v- 2638:2638 + v- _start ld-2.19.so 1 10074071 100.0 211135 100.0 + |- unknown unknown 1 13198 0.1 1 0.0 + >- _dl_start ld-2.19.so 1 1400980 13.9 19637 9.3 + >- _d_linit_internal ld-2.19.so 1 448152 4.4 11094 5.3 + v-__libc_start_main@plt ls 1 8211741 81.5 180397 85.4 + >- _dl_fixup ld-2.19.so 1 7607 0.1 108 0.1 + >- __cxa_atexit libc-2.19.so 1 11737 0.1 10 0.0 + >- __libc_csu_init ls 1 10354 0.1 10 0.0 + |- _setjmp libc-2.19.so 1 0 0.0 4 0.0 + v- main ls 1 8182043 99.6 180254 99.9 +</pre> +<h3>Points to note:</h3> +<ul> +<li>The top level is a command name (comm)</li> +<li>The next level is a thread (pid:tid)</li> +<li>Subsequent levels are functions</li> +<li>'Count' is the number of calls</li> +<li>'Time' is the elapsed time until the function returns</li> +<li>Percentages are relative to the level above</li> +<li>'Branch Count' is the total number of branches for that function and all functions that it calls +</ul> +<h3>Find</h3> +Ctrl-F displays a Find bar which finds function names by either an exact match or a pattern match. +The pattern matching symbols are ? for any character and * for zero or more characters. +<h2 id=calltree>1.2 Call Tree</h2> +The Call Tree report is very similar to the Context-Sensitive Call Graph, but the data is not aggregated. +Also the 'Count' column, which would be always 1, is replaced by the 'Call Time'. +<h2 id=allbranches>1.3 All branches</h2> +The All branches report displays all branches in chronological order. +Not all data is fetched immediately. More records can be fetched using the Fetch bar provided. +<h3>Disassembly</h3> +Open a branch to display disassembly. This only works if: +<ol> +<li>The disassembler is available. Currently, only Intel XED is supported - see <a href=#xed>Intel XED Setup</a></li> +<li>The object code is available. Currently, only the perf build ID cache is searched for object code. +The default directory ~/.debug can be overridden by setting environment variable PERF_BUILDID_DIR. +One exception is kcore where the DSO long name is used (refer dsos_view on the Tables menu), +or alternatively, set environment variable PERF_KCORE to the kcore file name.</li> +</ol> +<h4 id=xed>Intel XED Setup</h4> +To use Intel XED, libxed.so must be present. To build and install libxed.so: +<pre> +git clone https://github.com/intelxed/mbuild.git mbuild +git clone https://github.com/intelxed/xed +cd xed +./mfile.py --share +sudo ./mfile.py --prefix=/usr/local install +sudo ldconfig +</pre> +<h3>Instructions per Cycle (IPC)</h3> +If available, IPC information is displayed in columns 'insn_cnt', 'cyc_cnt' and 'IPC'. +<p><b>Intel PT note:</b> The information applies to the blocks of code ending with, and including, that branch. +Due to the granularity of timing information, the number of cycles for some code blocks will not be known. +In that case, 'insn_cnt', 'cyc_cnt' and 'IPC' are zero, but when 'IPC' is displayed it covers the period +since the previous displayed 'IPC'. +<h3>Find</h3> +Ctrl-F displays a Find bar which finds substrings by either an exact match or a regular expression match. +Refer to Python documentation for the regular expression syntax. +All columns are searched, but only currently fetched rows are searched. +<h2 id=selectedbranches>1.4 Selected branches</h2> +This is the same as the <a href=#allbranches>All branches</a> report but with the data reduced +by various selection criteria. A dialog box displays available criteria which are AND'ed together. +<h3>1.4.1 Time ranges</h3> +The time ranges hint text shows the total time range. Relative time ranges can also be entered in +ms, us or ns. Also, negative values are relative to the end of trace. Examples: +<pre> + 81073085947329-81073085958238 From 81073085947329 to 81073085958238 + 100us-200us From 100us to 200us + 10ms- From 10ms to the end + -100ns The first 100ns + -10ms- The last 10ms +</pre> +N.B. Due to the granularity of timestamps, there could be no branches in any given time range. +<h2 id=topcallsbyelapsedtime>1.5 Top calls by elapsed time</h2> +The Top calls by elapsed time report displays calls in descending order of time elapsed between when the function was called and when it returned. +The data is reduced by various selection criteria. A dialog box displays available criteria which are AND'ed together. +If not all data is fetched, a Fetch bar is provided. Ctrl-F displays a Find bar. +<h1 id=charts>2. Charts</h1> +<h2 id=timechartbycpu>2.1 Time chart by CPU</h2> +This chart displays context switch information when that data is available. Refer to context_switches_view on the Tables menu. +<h3>Features</h3> +<ol> +<li>Mouse over to highight the task and show the time</li> +<li>Drag the mouse to select a region and zoom by pushing the Zoom button</li> +<li>Go back and forward by pressing the arrow buttons</li> +<li>If call information is available, right-click to show a call tree opened to that task and time. +Note, the call tree may take some time to appear, and there may not be call information for the task or time selected. +</li> +</ol> +<h3>Important</h3> +The graph can be misleading in the following respects: +<ol> +<li>The graph shows the first task on each CPU as running from the beginning of the time range. +Because tracing might start on different CPUs at different times, that is not necessarily the case. +Refer to context_switches_view on the Tables menu to understand what data the graph is based upon.</li> +<li>Similarly, the last task on each CPU can be showing running longer than it really was. +Again, refer to context_switches_view on the Tables menu to understand what data the graph is based upon.</li> +<li>When the mouse is over a task, the highlighted task might not be visible on the legend without scrolling if the legend does not fit fully in the window</li> +</ol> +<h1 id=tables>3. Tables</h1> +The Tables menu shows all tables and views in the database. Most tables have an associated view +which displays the information in a more friendly way. Not all data for large tables is fetched +immediately. More records can be fetched using the Fetch bar provided. Columns can be sorted, +but that can be slow for large tables. +<p>There are also tables of database meta-information. +For SQLite3 databases, the sqlite_master table is included. +For PostgreSQL databases, information_schema.tables/views/columns are included. +<h3>Find</h3> +Ctrl-F displays a Find bar which finds substrings by either an exact match or a regular expression match. +Refer to Python documentation for the regular expression syntax. +All columns are searched, but only currently fetched rows are searched. +<p>N.B. Results are found in id order, so if the table is re-ordered, find-next and find-previous +will go to the next/previous result in id order, instead of display order. +""" + +# Help window + +class HelpWindow(QMdiSubWindow): + + def __init__(self, glb, parent=None): + super(HelpWindow, self).__init__(parent) + + self.text = QTextBrowser() + self.text.setHtml(glb_help_text) + self.text.setReadOnly(True) + self.text.setOpenExternalLinks(True) + + self.setWidget(self.text) + + AddSubWindow(glb.mainwindow.mdi_area, self, "Exported SQL Viewer Help") + +# Main window that only displays the help text + +class HelpOnlyWindow(QMainWindow): + + def __init__(self, parent=None): + super(HelpOnlyWindow, self).__init__(parent) + + self.setMinimumSize(200, 100) + self.resize(800, 600) + self.setWindowTitle("Exported SQL Viewer Help") + self.setWindowIcon(self.style().standardIcon(QStyle.SP_MessageBoxInformation)) + + self.text = QTextBrowser() + self.text.setHtml(glb_help_text) + self.text.setReadOnly(True) + self.text.setOpenExternalLinks(True) + + self.setCentralWidget(self.text) + +# PostqreSQL server version + +def PostqreSQLServerVersion(db): + query = QSqlQuery(db) + QueryExec(query, "SELECT VERSION()") + if query.next(): + v_str = query.value(0) + v_list = v_str.strip().split(" ") + if v_list[0] == "PostgreSQL" and v_list[2] == "on": + return v_list[1] + return v_str + return "Unknown" + +# SQLite version + +def SQLiteVersion(db): + query = QSqlQuery(db) + QueryExec(query, "SELECT sqlite_version()") + if query.next(): + return query.value(0) + return "Unknown" + +# About dialog + +class AboutDialog(QDialog): + + def __init__(self, glb, parent=None): + super(AboutDialog, self).__init__(parent) + + self.setWindowTitle("About Exported SQL Viewer") + self.setMinimumWidth(300) + + pyside_version = "1" if pyside_version_1 else "2" + + text = "<pre>" + text += "Python version: " + sys.version.split(" ")[0] + "\n" + text += "PySide version: " + pyside_version + "\n" + text += "Qt version: " + qVersion() + "\n" + if glb.dbref.is_sqlite3: + text += "SQLite version: " + SQLiteVersion(glb.db) + "\n" + else: + text += "PostqreSQL version: " + PostqreSQLServerVersion(glb.db) + "\n" + text += "</pre>" + + self.text = QTextBrowser() + self.text.setHtml(text) + self.text.setReadOnly(True) + self.text.setOpenExternalLinks(True) + + self.vbox = QVBoxLayout() + self.vbox.addWidget(self.text) + + self.setLayout(self.vbox) + +# Font resize + +def ResizeFont(widget, diff): + font = widget.font() + sz = font.pointSize() + font.setPointSize(sz + diff) + widget.setFont(font) + +def ShrinkFont(widget): + ResizeFont(widget, -1) + +def EnlargeFont(widget): + ResizeFont(widget, 1) + +# Unique name for sub-windows + +def NumberedWindowName(name, nr): + if nr > 1: + name += " <" + str(nr) + ">" + return name + +def UniqueSubWindowName(mdi_area, name): + nr = 1 + while True: + unique_name = NumberedWindowName(name, nr) + ok = True + for sub_window in mdi_area.subWindowList(): + if sub_window.name == unique_name: + ok = False + break + if ok: + return unique_name + nr += 1 + +# Add a sub-window + +def AddSubWindow(mdi_area, sub_window, name): + unique_name = UniqueSubWindowName(mdi_area, name) + sub_window.setMinimumSize(200, 100) + sub_window.resize(800, 600) + sub_window.setWindowTitle(unique_name) + sub_window.setAttribute(Qt.WA_DeleteOnClose) + sub_window.setWindowIcon(sub_window.style().standardIcon(QStyle.SP_FileIcon)) + sub_window.name = unique_name + mdi_area.addSubWindow(sub_window) + sub_window.show() + +# Main window + +class MainWindow(QMainWindow): + + def __init__(self, glb, parent=None): + super(MainWindow, self).__init__(parent) + + self.glb = glb + + self.setWindowTitle("Exported SQL Viewer: " + glb.dbname) + self.setWindowIcon(self.style().standardIcon(QStyle.SP_ComputerIcon)) + self.setMinimumSize(200, 100) + + self.mdi_area = QMdiArea() + self.mdi_area.setHorizontalScrollBarPolicy(Qt.ScrollBarAsNeeded) + self.mdi_area.setVerticalScrollBarPolicy(Qt.ScrollBarAsNeeded) + + self.setCentralWidget(self.mdi_area) + + menu = self.menuBar() + + file_menu = menu.addMenu("&File") + file_menu.addAction(CreateExitAction(glb.app, self)) + + edit_menu = menu.addMenu("&Edit") + edit_menu.addAction(CreateAction("&Copy", "Copy to clipboard", self.CopyToClipboard, self, QKeySequence.Copy)) + edit_menu.addAction(CreateAction("Copy as CS&V", "Copy to clipboard as CSV", self.CopyToClipboardCSV, self)) + edit_menu.addAction(CreateAction("&Find...", "Find items", self.Find, self, QKeySequence.Find)) + edit_menu.addAction(CreateAction("Fetch &more records...", "Fetch more records", self.FetchMoreRecords, self, [QKeySequence(Qt.Key_F8)])) + edit_menu.addAction(CreateAction("&Shrink Font", "Make text smaller", self.ShrinkFont, self, [QKeySequence("Ctrl+-")])) + edit_menu.addAction(CreateAction("&Enlarge Font", "Make text bigger", self.EnlargeFont, self, [QKeySequence("Ctrl++")])) + + reports_menu = menu.addMenu("&Reports") + if IsSelectable(glb.db, "calls"): + reports_menu.addAction(CreateAction("Context-Sensitive Call &Graph", "Create a new window containing a context-sensitive call graph", self.NewCallGraph, self)) + + if IsSelectable(glb.db, "calls", "WHERE parent_id >= 0"): + reports_menu.addAction(CreateAction("Call &Tree", "Create a new window containing a call tree", self.NewCallTree, self)) + + self.EventMenu(GetEventList(glb.db), reports_menu) + + if IsSelectable(glb.db, "calls"): + reports_menu.addAction(CreateAction("&Top calls by elapsed time", "Create a new window displaying top calls by elapsed time", self.NewTopCalls, self)) + + if IsSelectable(glb.db, "context_switches"): + charts_menu = menu.addMenu("&Charts") + charts_menu.addAction(CreateAction("&Time chart by CPU", "Create a new window displaying time charts by CPU", self.TimeChartByCPU, self)) + + self.TableMenu(GetTableList(glb), menu) + + self.window_menu = WindowMenu(self.mdi_area, menu) + + help_menu = menu.addMenu("&Help") + help_menu.addAction(CreateAction("&Exported SQL Viewer Help", "Helpful information", self.Help, self, QKeySequence.HelpContents)) + help_menu.addAction(CreateAction("&About Exported SQL Viewer", "About this application", self.About, self)) + + def Try(self, fn): + win = self.mdi_area.activeSubWindow() + if win: + try: + fn(win.view) + except AttributeError: + pass + + def CopyToClipboard(self): + self.Try(CopyCellsToClipboardHdr) + + def CopyToClipboardCSV(self): + self.Try(CopyCellsToClipboardCSV) + + def Find(self): + win = self.mdi_area.activeSubWindow() + if win: + try: + win.find_bar.Activate() + except AttributeError: + pass + + def FetchMoreRecords(self): + win = self.mdi_area.activeSubWindow() + if win: + try: + win.fetch_bar.Activate() + except AttributeError: + pass + + def ShrinkFont(self): + self.Try(ShrinkFont) + + def EnlargeFont(self): + self.Try(EnlargeFont) + + def EventMenu(self, events, reports_menu): + branches_events = 0 + for event in events: + event = event.split(":")[0] + if event == "branches": + branches_events += 1 + dbid = 0 + for event in events: + dbid += 1 + event = event.split(":")[0] + if event == "branches": + label = "All branches" if branches_events == 1 else "All branches " + "(id=" + dbid + ")" + reports_menu.addAction(CreateAction(label, "Create a new window displaying branch events", lambda a=None,x=dbid: self.NewBranchView(x), self)) + label = "Selected branches" if branches_events == 1 else "Selected branches " + "(id=" + dbid + ")" + reports_menu.addAction(CreateAction(label, "Create a new window displaying branch events", lambda a=None,x=dbid: self.NewSelectedBranchView(x), self)) + + def TimeChartByCPU(self): + TimeChartByCPUWindow(self.glb, self) + + def TableMenu(self, tables, menu): + table_menu = menu.addMenu("&Tables") + for table in tables: + table_menu.addAction(CreateAction(table, "Create a new window containing a table view", lambda a=None,t=table: self.NewTableView(t), self)) + + def NewCallGraph(self): + CallGraphWindow(self.glb, self) + + def NewCallTree(self): + CallTreeWindow(self.glb, self) + + def NewTopCalls(self): + dialog = TopCallsDialog(self.glb, self) + ret = dialog.exec_() + if ret: + TopCallsWindow(self.glb, dialog.report_vars, self) + + def NewBranchView(self, event_id): + BranchWindow(self.glb, event_id, ReportVars(), self) + + def NewSelectedBranchView(self, event_id): + dialog = SelectedBranchDialog(self.glb, self) + ret = dialog.exec_() + if ret: + BranchWindow(self.glb, event_id, dialog.report_vars, self) + + def NewTableView(self, table_name): + TableWindow(self.glb, table_name, self) + + def Help(self): + HelpWindow(self.glb, self) + + def About(self): + dialog = AboutDialog(self.glb, self) + dialog.exec_() + +def TryOpen(file_name): + try: + return open(file_name, "rb") + except OSError: + return None + +def Is64Bit(f): + result = sizeof(c_void_p) + # ELF support only + pos = f.tell() + f.seek(0) + header = f.read(7) + f.seek(pos) + magic = header[0:4] + if sys.version_info[0] == 2: + eclass = ord(header[4]) + encoding = ord(header[5]) + version = ord(header[6]) + else: + eclass = header[4] + encoding = header[5] + version = header[6] + if magic == chr(127) + "ELF" and eclass > 0 and eclass < 3 and encoding > 0 and encoding < 3 and version == 1: + result = True if eclass == 2 else False + return result + +# Global data + +class Glb(): + + def __init__(self, dbref, db, dbname): + self.dbref = dbref + self.db = db + self.dbname = dbname + self.home_dir = os.path.expanduser("~") + self.buildid_dir = os.getenv("PERF_BUILDID_DIR") + if self.buildid_dir: + self.buildid_dir += "/.build-id/" + else: + self.buildid_dir = self.home_dir + "/.debug/.build-id/" + self.app = None + self.mainwindow = None + self.instances_to_shutdown_on_exit = weakref.WeakSet() + try: + self.disassembler = LibXED() + self.have_disassembler = True + except (OSError, RuntimeError, AttributeError): + self.have_disassembler = False + self.host_machine_id = 0 + self.host_start_time = 0 + self.host_finish_time = 0 + + def FileFromBuildId(self, build_id): + file_name = self.buildid_dir + build_id[0:2] + "/" + build_id[2:] + "/elf" + return TryOpen(file_name) + + def FileFromNamesAndBuildId(self, short_name, long_name, build_id): + # Assume current machine i.e. no support for virtualization + if short_name[0:7] == "[kernel" and os.path.basename(long_name) == "kcore": + file_name = os.getenv("PERF_KCORE") + f = TryOpen(file_name) if file_name else None + if f: + return f + # For now, no special handling if long_name is /proc/kcore + f = TryOpen(long_name) + if f: + return f + f = self.FileFromBuildId(build_id) + if f: + return f + return None + + def AddInstanceToShutdownOnExit(self, instance): + self.instances_to_shutdown_on_exit.add(instance) + + # Shutdown any background processes or threads + def ShutdownInstances(self): + for x in self.instances_to_shutdown_on_exit: + try: + x.Shutdown() + except AttributeError: + pass + + def GetHostMachineId(self): + query = QSqlQuery(self.db) + QueryExec(query, "SELECT id FROM machines WHERE pid = -1") + if query.next(): + self.host_machine_id = query.value(0) + else: + self.host_machine_id = 0 + return self.host_machine_id + + def HostMachineId(self): + if self.host_machine_id: + return self.host_machine_id + return self.GetHostMachineId() + + def SelectValue(self, sql): + query = QSqlQuery(self.db) + try: + QueryExec(query, sql) + except RuntimeError: + return None + if query.next(): + return Decimal(query.value(0)) + return None + + def SwitchesMinTime(self, machine_id): + return self.SelectValue("SELECT time" + " FROM context_switches" + " WHERE time != 0 AND machine_id = " + str(machine_id) + + " ORDER BY id LIMIT 1") + + def SwitchesMaxTime(self, machine_id): + return self.SelectValue("SELECT time" + " FROM context_switches" + " WHERE time != 0 AND machine_id = " + str(machine_id) + + " ORDER BY id DESC LIMIT 1") + + def SamplesMinTime(self, machine_id): + return self.SelectValue("SELECT time" + " FROM samples" + " WHERE time != 0 AND machine_id = " + str(machine_id) + + " ORDER BY id LIMIT 1") + + def SamplesMaxTime(self, machine_id): + return self.SelectValue("SELECT time" + " FROM samples" + " WHERE time != 0 AND machine_id = " + str(machine_id) + + " ORDER BY id DESC LIMIT 1") + + def CallsMinTime(self, machine_id): + return self.SelectValue("SELECT calls.call_time" + " FROM calls" + " INNER JOIN threads ON threads.id = calls.thread_id" + " WHERE calls.call_time != 0 AND threads.machine_id = " + str(machine_id) + + " ORDER BY calls.id LIMIT 1") + + def CallsMaxTime(self, machine_id): + return self.SelectValue("SELECT calls.return_time" + " FROM calls" + " INNER JOIN threads ON threads.id = calls.thread_id" + " WHERE calls.return_time != 0 AND threads.machine_id = " + str(machine_id) + + " ORDER BY calls.return_time DESC LIMIT 1") + + def GetStartTime(self, machine_id): + t0 = self.SwitchesMinTime(machine_id) + t1 = self.SamplesMinTime(machine_id) + t2 = self.CallsMinTime(machine_id) + if t0 is None or (not(t1 is None) and t1 < t0): + t0 = t1 + if t0 is None or (not(t2 is None) and t2 < t0): + t0 = t2 + return t0 + + def GetFinishTime(self, machine_id): + t0 = self.SwitchesMaxTime(machine_id) + t1 = self.SamplesMaxTime(machine_id) + t2 = self.CallsMaxTime(machine_id) + if t0 is None or (not(t1 is None) and t1 > t0): + t0 = t1 + if t0 is None or (not(t2 is None) and t2 > t0): + t0 = t2 + return t0 + + def HostStartTime(self): + if self.host_start_time: + return self.host_start_time + self.host_start_time = self.GetStartTime(self.HostMachineId()) + return self.host_start_time + + def HostFinishTime(self): + if self.host_finish_time: + return self.host_finish_time + self.host_finish_time = self.GetFinishTime(self.HostMachineId()) + return self.host_finish_time + + def StartTime(self, machine_id): + if machine_id == self.HostMachineId(): + return self.HostStartTime() + return self.GetStartTime(machine_id) + + def FinishTime(self, machine_id): + if machine_id == self.HostMachineId(): + return self.HostFinishTime() + return self.GetFinishTime(machine_id) + +# Database reference + +class DBRef(): + + def __init__(self, is_sqlite3, dbname): + self.is_sqlite3 = is_sqlite3 + self.dbname = dbname + self.TRUE = "TRUE" + self.FALSE = "FALSE" + # SQLite prior to version 3.23 does not support TRUE and FALSE + if self.is_sqlite3: + self.TRUE = "1" + self.FALSE = "0" + + def Open(self, connection_name): + dbname = self.dbname + if self.is_sqlite3: + db = QSqlDatabase.addDatabase("QSQLITE", connection_name) + else: + db = QSqlDatabase.addDatabase("QPSQL", connection_name) + opts = dbname.split() + for opt in opts: + if "=" in opt: + opt = opt.split("=") + if opt[0] == "hostname": + db.setHostName(opt[1]) + elif opt[0] == "port": + db.setPort(int(opt[1])) + elif opt[0] == "username": + db.setUserName(opt[1]) + elif opt[0] == "password": + db.setPassword(opt[1]) + elif opt[0] == "dbname": + dbname = opt[1] + else: + dbname = opt + + db.setDatabaseName(dbname) + if not db.open(): + raise RuntimeError("Failed to open database " + dbname + " error: " + db.lastError().text()) + return db, dbname + +# Main + +def Main(): + usage_str = "exported-sql-viewer.py [--pyside-version-1] <database name>\n" \ + " or: exported-sql-viewer.py --help-only" + ap = argparse.ArgumentParser(usage = usage_str, add_help = False) + ap.add_argument("--pyside-version-1", action='store_true') + ap.add_argument("dbname", nargs="?") + ap.add_argument("--help-only", action='store_true') + args = ap.parse_args() + + if args.help_only: + app = QApplication(sys.argv) + mainwindow = HelpOnlyWindow() + mainwindow.show() + err = app.exec_() + sys.exit(err) + + dbname = args.dbname + if dbname is None: + ap.print_usage() + print("Too few arguments") + sys.exit(1) + + is_sqlite3 = False + try: + f = open(dbname, "rb") + if f.read(15) == b'SQLite format 3': + is_sqlite3 = True + f.close() + except OSError: + pass + + dbref = DBRef(is_sqlite3, dbname) + db, dbname = dbref.Open("main") + glb = Glb(dbref, db, dbname) + app = QApplication(sys.argv) + glb.app = app + mainwindow = MainWindow(glb) + glb.mainwindow = mainwindow + mainwindow.show() + err = app.exec_() + glb.ShutdownInstances() + db.close() + sys.exit(err) + +if __name__ == "__main__": + Main() diff --git a/tools/perf/python/failed-syscalls-by-pid.py b/tools/perf/python/failed-syscalls-by-pid.py new file mode 100755 index 000000000000..51824a929ea4 --- /dev/null +++ b/tools/perf/python/failed-syscalls-by-pid.py @@ -0,0 +1,161 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +""" +Displays system-wide failed system call totals, broken down by pid. +If a [comm] or [pid] arg is specified, only syscalls called by it are displayed. + +Ported from tools/perf/scripts/python/failed-syscalls-by-pid.py +""" +from __future__ import annotations + +import argparse +from collections import defaultdict +import os +import sys +from typing import Optional +import perf + + +def strerror(nr: int, e_machine: Optional[int] = None) -> str: + """Return error string for a given errno, accounting for target e_machine.""" + return perf.arch_strerrno(nr, e_machine) or f"Unknown {nr} errno" + + +class SyscallAnalyzer: + """Analyzes failed syscalls and aggregates counts.""" + + def __init__(self, for_comm: Optional[str] = None, for_pid: Optional[int] = None): + self.for_comm = for_comm + self.for_pid = for_pid + self.session: Optional[perf.session] = None + self.syscalls: dict[tuple[str, int, int, int], int] = defaultdict(int) + machine = os.uname().machine + self.host_64_bit = ("64" in machine or machine in ("s390x", "alpha") + or sys.maxsize > 0xffffffff) + + def process_event(self, sample: perf.sample_event) -> None: + """Process raw_syscalls:sys_exit and syscalls:sys_exit events.""" + event_name = str(sample.evsel) + if "raw_syscalls:sys_exit" not in event_name and "syscalls:sys_exit" not in event_name: + return + + pid = sample.sample_tid + comm = "Unknown" + if hasattr(self, 'session') and self.session: + try: + thread = self.session.find_thread(sample.sample_pid, pid) + if thread: + comm = thread.comm() or "Unknown" + except (OSError, ValueError, KeyError, RuntimeError, TypeError, AttributeError): + pass + + if self.for_comm is not None and comm != self.for_comm: + return + if self.for_pid is not None and pid != self.for_pid: + return + + ret = getattr(sample, "ret", 0) + is_64_bit = (getattr(self.session, "is_64_bit", self.host_64_bit) + if self.session else self.host_64_bit) + # Tracepoint fields may be exposed as unsigned 32-bit or 64-bit integers. + # Guard the 32-bit 0xfffff000..0xffffffff conversion with 'not is_64_bit' so valid + # 64-bit user-space addresses in that range (e.g. mmap/brk) are not misclassified. + if ret > 0: + if not is_64_bit and 0xfffff000 <= ret <= 0xffffffff: + ret -= 0x100000000 + elif ret >= 0xfffffffffffff000: + ret -= 0x10000000000000000 + + # Linux kernel syscalls only return error codes in the [-4095, -1] (-MAX_ERRNO) + # range; other negative signed values are valid unsigned user-space pointers. + if -4095 <= ret < 0: + syscall_id = getattr(sample, "__syscall_nr", -1) + if not (0 <= (syscall_id & ~0x40000000) <= 0xffff): + syscall_id = getattr(sample, "nr", -1) + if not (0 <= (syscall_id & ~0x40000000) <= 0xffff): + syscall_id = getattr(sample, "sys_id", -1) + if not (0 <= (syscall_id & ~0x40000000) <= 0xffff): + syscall_id = getattr(sample, "id", -1) + + # Mask out the x86_64 x32 ABI bit (__X32_SYSCALL_BIT = 0x40000000). + raw_sc_id = syscall_id & ~0x40000000 + if 0 <= raw_sc_id <= 0xffff: + self.syscalls[(comm, pid, syscall_id, ret)] += 1 + + def print_summary(self) -> None: + """Print aggregated statistics.""" + if self.for_comm is not None: + print(f"\nsyscall errors for {self.for_comm}:\n") + elif self.for_pid is not None: + print(f"\nsyscall errors for PID {self.for_pid}:\n") + else: + print("\nsyscall errors:\n") + + print(f"{'comm [pid]':<30} {'count':>10}") + print(f"{'-' * 30:<30} {'-' * 10:>10}") + + sorted_keys = sorted( + self.syscalls.keys(), + key=lambda k: (k[0], k[1], k[2], -self.syscalls[k]) + ) + current_comm_pid = None + current_syscall = None + emach = getattr(self.session, "e_machine", 0) or 0 + for comm, pid, syscall_id, ret in sorted_keys: + if current_comm_pid != (comm, pid): + print(f"\n{comm} [{pid}]") + current_comm_pid = (comm, pid) + current_syscall = None + if current_syscall != syscall_id: + raw_sc_id = syscall_id & ~0x40000000 + try: + if emach: + name = perf.syscall_name(raw_sc_id, emach) or str(syscall_id) + else: + name = perf.syscall_name(raw_sc_id) or str(syscall_id) + except AttributeError: + name = str(syscall_id) + print(f" syscall: {name:<16}") + current_syscall = syscall_id + err_str = strerror(ret, emach) + count = self.syscalls[(comm, pid, syscall_id, ret)] + print(f" err = {err_str:<20} {count:10d}") + + +if __name__ == "__main__": + ap = argparse.ArgumentParser( + description="Displays system-wide failed system call totals, " + "broken down by pid.") + ap.add_argument("-i", "--input", default="perf.data", + help="Input file name") + ap.add_argument("filter", nargs="?", help="COMM or PID to filter by") + args = ap.parse_args() + + F_COMM = None + F_PID = None + + if args.filter: + try: + F_PID = int(args.filter) + except ValueError: + F_COMM = args.filter + + analyzer = SyscallAnalyzer(F_COMM, F_PID) + session = None + + try: + print("Press control+C to stop and show the summary") + session = perf.session(perf.data(args.input), sample=analyzer.process_event) + analyzer.session = session + session.process_events() + analyzer.print_summary() + except KeyboardInterrupt: + analyzer.print_summary() + except (OSError, ValueError, KeyError, RuntimeError, TypeError, AttributeError) as e: + print(f"Error processing events: {e}") + sys.exit(1) + finally: + # Break reference cycles between perf.session and analyzer.process_event + # because perf.session is a C extension type without cyclic GC (tp_traverse). + analyzer.session = None + session = None diff --git a/tools/perf/python/failed-syscalls.py b/tools/perf/python/failed-syscalls.py new file mode 100755 index 000000000000..6d1fa4e5ef89 --- /dev/null +++ b/tools/perf/python/failed-syscalls.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +"""Failed system call counts.""" +from __future__ import annotations + +import argparse +from collections import defaultdict +import sys +from typing import Optional +import perf + +class FailedSyscalls: + """Tracks and displays failed system call totals.""" + def __init__(self, comm: Optional[str] = None) -> None: + self.failed_syscalls: dict[str, int] = defaultdict(int) + self.for_comm = comm + self.session: Optional[perf.session] = None + + def process_event(self, sample: perf.sample_event) -> None: + """Process sys_exit events.""" + event_name = str(sample.evsel) + if not event_name.startswith("evsel(syscalls:sys_exit") and \ + not event_name.startswith("evsel(raw_syscalls:sys_exit"): + return + + try: + ret = sample.ret + except AttributeError: + print("ERROR: tracepoint fields missed", file=sys.stderr) + sys.exit(1) + + if ret > 0: + if ret >= 0xfffffffffffff000: # 64-bit negative errors + ret -= 0x10000000000000000 + elif 0xfffff000 <= ret <= 0xffffffff: # 32-bit negative errors + assert self.session is not None + if not self.session.is_64_bit: + ret -= 0x100000000 + + # Linux kernel syscalls only return error codes in the [-4095, -1] (-MAX_ERRNO) + # range; other negative signed values are valid unsigned user-space addresses. + if not -4095 <= ret < 0: + return + + assert self.session is not None + try: + thread = self.session.find_thread(sample.sample_pid, sample.sample_tid) + comm = (thread.comm() if thread else None) or "unknown" + except (TypeError, AttributeError): + # find_thread returns None when the thread isn't known. + comm = "unknown" + + if self.for_comm and comm != self.for_comm: + return + + self.failed_syscalls[comm] += 1 + + def print_totals(self) -> None: + """Print summary table.""" + print("\nfailed syscalls by comm:\n") + print(f"{'comm':<20s} {'# errors':>10s}") + print(f"{'-'*20} {'-'*10}") + + for comm, val in sorted(self.failed_syscalls.items(), + key=lambda kv: (-kv[1], kv[0])): + print(f"{comm:<20s} {val:10d}") + + def run(self, input_file: str) -> None: + """Run the session.""" + self.session = perf.session(perf.data(input_file), sample=self.process_event) + try: + self.session.process_events() + finally: + # Break the reference cycle between perf.session and self.process_event + # because perf.session lacks cyclic GC support (tp_traverse). + self.session = None + self.print_totals() + +def main() -> None: + """Main function.""" + parser = argparse.ArgumentParser(description="Trace failed syscalls") + parser.add_argument("comm", nargs="?", help="Filter by command name") + parser.add_argument("-i", "--input", default="perf.data", help="Input file") + args = parser.parse_args() + + analyzer = FailedSyscalls(args.comm) + try: + analyzer.run(args.input) + except IOError as e: + print(e, file=sys.stderr) + sys.exit(1) + +if __name__ == "__main__": + main() diff --git a/tools/perf/python/flamegraph.py b/tools/perf/python/flamegraph.py new file mode 100755 index 000000000000..1b06a11fa86c --- /dev/null +++ b/tools/perf/python/flamegraph.py @@ -0,0 +1,283 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +""" +flamegraph.py - create flame graphs from perf samples using perf python module +""" +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import subprocess +import sys +import urllib.request +from typing import Dict, Optional, Union +import perf + +MINIMAL_HTML = """<head> + <link rel="stylesheet" type="text/css" href="https://cdn.jsdelivr.net/npm/d3-flame-graph@4.1.3/dist/d3-flamegraph.css" integrity="sha256-fXXWjLZigZo3QBJIhjEA3Z2zUfS+vzMQhF2Rlvurfio=" crossorigin="anonymous"> +</head> +<body> + <div id="chart"></div> + <script type="text/javascript" src="https://d3js.org/d3.v7.js" integrity="sha256-DA1YviEGwU4kL8dbwJoXRngAYwdRWAZl9ulVKgwGTqI=" crossorigin="anonymous"></script> + <script type="text/javascript" src="https://cdn.jsdelivr.net/npm/d3-flame-graph@4.1.3/dist/d3-flamegraph.min.js" integrity="sha256-GEbx5jbtG9LDL8h9HiTWOv9+Txhim0u9gZNR8J6iI1M=" crossorigin="anonymous"></script> + <script type="text/javascript"> + const stacks = [/** @flamegraph_json **/]; + // Note, options is unused. + const options = [/** @options_json **/]; + + var chart = flamegraph(); + d3.select("#chart") + .datum(stacks[0]) + .call(chart); + </script> +</body> +""" + +class Node: + """A node in the flame graph tree.""" + def __init__(self, name: str, libtype: str): + self.name = name + self.libtype = libtype + self.value: int = 0 + self.children: dict[str, Node] = {} + + def to_json(self) -> Dict[str, Union[str, int, list[Dict]]]: + """Convert the node to a JSON-serializable dictionary.""" + return { + "n": self.name, + "l": self.libtype, + "v": self.value, + "c": [x.to_json() for x in self.children.values()] + } + + +class FlameGraphCLI: + """Command-line interface for generating flame graphs.""" + def __init__(self, args): + self.args = args + self.stack = Node("all", "root") + self.session = None + + @staticmethod + def get_libtype_from_dso(dso: Optional[str]) -> str: + """Determine the library type from the DSO name.""" + if dso and (dso == "[kernel.kallsyms]" or dso.endswith("/vmlinux") or dso == "[kernel]"): + return "kernel" + return "" + + @staticmethod + def find_or_create_node(node: Node, name: str, libtype: str) -> Node: + """Find a child node with the given name or create a new one.""" + if name in node.children: + return node.children[name] + child = Node(name, libtype) + node.children[name] = child + return child + + def process_event(self, sample) -> None: + """Process a single perf sample event.""" + if self.args.event_name and self.args.event_name not in str(sample.evsel): + return + + pid = sample.sample_pid + dso_type = "" + try: + thread = self.session.find_thread(sample.sample_pid, sample.sample_tid) + comm = (thread.comm() if thread else None) or "[unknown]" + except (OSError, ValueError, KeyError, RuntimeError, TypeError, AttributeError): + comm = "[unknown]" + + if pid == 0: + comm = comm if comm != "[unknown]" else "swapper" + dso_type = "kernel" + else: + comm = f"{comm} ({pid})" + + node = self.find_or_create_node(self.stack, comm, dso_type) + + callchain = sample.callchain + if callchain: + # We want to traverse from root to leaf. + # perf callchain iterator gives leaf to root. + # We collect them and reverse. + frames = list(callchain) + for entry in reversed(frames): + name = entry.symbol or "[unknown]" + libtype = self.get_libtype_from_dso(entry.dso) + node = self.find_or_create_node(node, name, libtype) + else: + # Fallback if no callchain + name = (sample.symbol or '[unknown]') + libtype = self.get_libtype_from_dso((sample.dso or '[unknown]')) + node = self.find_or_create_node(node, name, libtype) + + node.value += 1 + + def get_report_header(self) -> str: + """Get the header from the perf report.""" + try: + input_file = self.args.input or "perf.data" + if input_file == "-": + return "" + mode = os.stat(input_file).st_mode + import stat + if stat.S_ISFIFO(mode) or stat.S_ISCHR(mode): + return "" + output = subprocess.check_output(["perf", "report", "--header-only", "-i", input_file]) + result = output.decode("utf-8") + if self.args.event_name: + result += "\nFocused event: " + self.args.event_name + return result + except (OSError, ValueError, KeyError, RuntimeError, TypeError, AttributeError, + subprocess.CalledProcessError): + return "" + + def run(self) -> None: + """Run the flame graph generation.""" + input_file = self.args.input or "perf.data" + if input_file != "-" and not os.path.exists(input_file): + print(f"Error: {input_file} not found. (try 'perf record' first)", file=sys.stderr) + sys.exit(1) + + try: + self.session = perf.session(perf.data(input_file), + sample=self.process_event) + except (OSError, ValueError, KeyError, RuntimeError, TypeError, AttributeError) as e: + print(f"Error opening session: {e}", file=sys.stderr) + sys.exit(1) + + try: + self.session.process_events() + finally: + # perf.session is a C extension type that holds a strong reference to the + # bound method self.process_event (which references self) and does not + # implement cyclic GC (tp_traverse). Explicitly clear self.session to break + # the reference cycle and free the underlying session resources deterministically. + self.session = None + + stacks_json = json.dumps(self.stack, default=lambda x: x.to_json()) + # Escape HTML special characters to prevent XSS + stacks_json = stacks_json.replace("<", "\\u003c") \ + .replace(">", "\\u003e").replace("&", "\\u0026") + + if self.args.format == "html": + report_header = self.get_report_header() + options = { + "colorscheme": self.args.colorscheme, + "context": report_header + } + options_json = json.dumps(options) + options_json = options_json.replace("<", "\\u003c") \ + .replace(">", "\\u003e").replace("&", "\\u0026") + + template = self.args.template + template_sha256sum = None + output_str = None + + if not os.path.isfile(template): + if template.startswith("http://") or template.startswith("https://"): + if not self.args.allow_download: + print("Warning: Downloading templates is disabled. " + "Use --allow-download.", file=sys.stderr) + template = None + else: + print(f"Warning: Template file '{template}' not found.", file=sys.stderr) + if self.args.allow_download: + print("Using default CDN template.", file=sys.stderr) + template = ( + "https://cdn.jsdelivr.net/npm/d3-flame-graph@4.1.3/dist/templates/" + "d3-flamegraph-base.html" + ) + template_sha256sum = ( + "f6a4aa7edffda4fb9bd71eb0eb75bc44d0bb34cd9efbd053f6095bc5c28d702b" + ) + else: + template = None + + use_minimal = False + try: + if not template: + use_minimal = True + elif template.startswith(("http://", "https://")): + with urllib.request.urlopen(template) as url_template: + output_str = "".join([l.decode("utf-8") for l in url_template.readlines()]) + else: + with open(template, "r", encoding="utf-8") as f: + output_str = f.read() + except (OSError, ValueError, KeyError, RuntimeError, TypeError, AttributeError) as err: + print(f"Error reading template {template}: {err}\n", file=sys.stderr) + use_minimal = True + + if use_minimal: + print("Using internal minimal HTML that refers to d3's web site. JavaScript " + + "loaded this way from a local file may be blocked unless your " + + "browser has relaxed permissions. Run with '--allow-download' to fetch " + + "the full D3 HTML template.", file=sys.stderr) + output_str = MINIMAL_HTML + + elif template_sha256sum: + assert output_str is not None + download_sha256sum = hashlib.sha256( + output_str.encode("utf-8") + ).hexdigest() + if download_sha256sum != template_sha256sum: + s = None + while s not in ["y", "n"]: + try: + s = input(f"""Unexpected template sha256sum. +{download_sha256sum} != {template_sha256sum}, for: +{template} +continue?[yn] """).lower() + except EOFError: + s = "n" + if s == "n": + sys.exit(1) + + assert output_str is not None + replacements = { + "/** @options_json **/": options_json, + "/** @flamegraph_json **/": stacks_json, + } + # Use a lambda replacement so re.sub does not interpret backslashes + # in the JSON strings (such as \n or \u003c) as regex escape sequences. + output_str = re.sub( + r"/\*\* @(?:options_json|flamegraph_json) \*\*/", + lambda m: replacements[m.group(0)], + output_str, + ) + output_fn = self.args.output or "flamegraph.html" + else: + output_str = stacks_json + output_fn = self.args.output or "stacks.json" + + if output_fn == "-": + with open(sys.stdout.fileno(), "w", encoding="utf-8", closefd=False) as out: + out.write(output_str) + else: + print(f"dumping data to {output_fn}") + with open(output_fn, "w", encoding="utf-8") as out: + out.write(output_str) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Create flame graphs using perf python module.") + parser.add_argument("-f", "--format", default="html", choices=["json", "html"], + help="output file format") + parser.add_argument("-o", "--output", help="output file name") + parser.add_argument("--template", + default="/usr/share/d3-flame-graph/d3-flamegraph-base.html", + help="path to flame graph HTML template") + parser.add_argument("--colorscheme", default="blue-green", + help="flame graph color scheme", choices=["blue-green", "orange"]) + parser.add_argument("-i", "--input", help="input perf.data file") + parser.add_argument("--allow-download", default=False, action="store_true", + help="allow unprompted downloading of HTML template") + parser.add_argument("-e", "--event", default="", dest="event_name", type=str, + help="specify the event to generate flamegraph for") + + cli_args = parser.parse_args() + cli = FlameGraphCLI(cli_args) + cli.run() diff --git a/tools/perf/python/futex-contention.py b/tools/perf/python/futex-contention.py new file mode 100755 index 000000000000..097663d929b7 --- /dev/null +++ b/tools/perf/python/futex-contention.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +"""Measures futex contention.""" +from __future__ import annotations + +import argparse +from collections import defaultdict +from typing import Dict, Tuple +import perf + +class LockStats: + """Aggregate lock contention information.""" + def __init__(self) -> None: + self.count = 0 + self.total_time = 0 + self.min_time = 0 + self.max_time = 0 + + def add(self, duration: int) -> None: + """Add a new duration measurement.""" + self.count += 1 + self.total_time += duration + if self.count == 1: + self.min_time = duration + self.max_time = duration + else: + self.min_time = min(self.min_time, duration) + self.max_time = max(self.max_time, duration) + + def avg(self) -> float: + """Return average duration.""" + return self.total_time / self.count if self.count > 0 else 0.0 + +process_names: Dict[int, str] = {} +start_times: Dict[int, Tuple[int, int]] = {} +session = None +durations: Dict[Tuple[int, int], LockStats] = defaultdict(LockStats) + +FUTEX_WAIT = 0 +FUTEX_PRIVATE_FLAG = 128 +FUTEX_CLOCK_REALTIME = 256 +# Mask out FUTEX_PRIVATE_FLAG and FUTEX_CLOCK_REALTIME so variants such as +# FUTEX_WAIT_PRIVATE (0 | 128) match the base FUTEX_WAIT command. +FUTEX_CMD_MASK = ~(FUTEX_PRIVATE_FLAG | FUTEX_CLOCK_REALTIME) + + +def handle_start(pid: int, tid: int, uaddr: int, op: int, start_time: int) -> None: + """Handle a futex sys_enter event.""" + if (op & FUTEX_CMD_MASK) != FUTEX_WAIT: + return + try: + if session: + process = session.find_thread(pid, tid) + if process: + process_names[tid] = process.comm() or "unknown" + except (TypeError, AttributeError): + pass + if tid not in process_names: + process_names[tid] = "unknown" + + start_times[tid] = (uaddr, start_time) + +def handle_end(tid: int, end_time: int) -> None: + """Handle a futex sys_exit event.""" + if tid not in start_times: + return + (uaddr, start_time) = start_times[tid] + del start_times[tid] + durations[(tid, uaddr)].add(end_time - start_time) + +def process_event(sample: perf.sample_event) -> None: + """Process a single sample event.""" + event_name = str(sample.evsel) + if event_name.startswith("evsel(") and event_name.endswith(")"): + event_name = event_name[6:-1] + if event_name.startswith("syscalls:sys_enter_futex"): + uaddr = getattr(sample, "uaddr", None) + op = getattr(sample, "op", None) + if uaddr is None or op is None: + return # Tracepoint fields missing, skip silent attribution to 0 + handle_start(getattr(sample, "sample_pid", sample.sample_tid), + sample.sample_tid, uaddr, op, sample.sample_time) + elif event_name.startswith("syscalls:sys_exit_futex"): + handle_end(sample.sample_tid, sample.sample_time) + + +if __name__ == "__main__": + ap = argparse.ArgumentParser(description="Measure futex contention") + ap.add_argument("-i", "--input", default="perf.data", help="Input file name") + args = ap.parse_args() + + try: + session = perf.session(perf.data(args.input), sample=process_event) + try: + session.process_events() + except KeyboardInterrupt: + pass + + for ((t, u), stats) in sorted(durations.items()): + avg_ns = stats.avg() + print(f"{process_names.get(t, 'unknown')}[{t}] lock {u:x} contended " + f"{stats.count} times, {avg_ns:.0f} avg ns " + f"[max: {stats.max_time} ns, min {stats.min_time} ns]") + finally: + # Break the reference cycle between the global session and the + # process_event callback so the C perf.session object is freed. + session = None diff --git a/tools/perf/python/gecko.py b/tools/perf/python/gecko.py new file mode 100755 index 000000000000..3427e0770156 --- /dev/null +++ b/tools/perf/python/gecko.py @@ -0,0 +1,411 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +""" +gecko.py - Convert perf record output to Firefox's gecko profile format +""" +from __future__ import annotations + +import argparse +import functools +import json +import os +import sys +import tempfile +import threading +import urllib.parse +import uuid +import webbrowser +from dataclasses import dataclass, field +from http.server import HTTPServer, SimpleHTTPRequestHandler +from typing import Dict, List, NamedTuple, Optional, Tuple + +import perf + + +# https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L156 +class Frame(NamedTuple): + """A single stack frame in the gecko profile format.""" + string_id: int + relevantForJS: bool + innerWindowID: int + implementation: None + optimizations: None + line: None + column: None + category: int + subcategory: Optional[int] + + +# https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L216 +class Stack(NamedTuple): + """A single stack in the gecko profile format.""" + prefix_id: Optional[int] + frame_id: int + + +# https://github.com/firefox-devtools/profiler/blob/53970305b51b9b472e26d7457fee1d66cd4e2737/src/types/gecko-profile.js#L90 +class Sample(NamedTuple): + """A single sample in the gecko profile format.""" + stack_id: Optional[int] + time_ms: float + responsiveness: int + + +@dataclass +class Tables: + """Interned tables for the gecko profile format.""" + frame_table: List[Frame] = field(default_factory=list) + string_table: List[str] = field(default_factory=list) + string_map: Dict[str, int] = field(default_factory=dict) + stack_table: List[Stack] = field(default_factory=list) + stack_map: Dict[Tuple[Optional[int], int], int] = field(default_factory=dict) + frame_map: Dict[str, int] = field(default_factory=dict) + + +@dataclass +class Thread: + """A builder for a profile of the thread.""" + comm: str + pid: int + tid: int + user_category: int + kernel_category: int + samples: List[Sample] = field(default_factory=list) + tables: Tables = field(default_factory=Tables) + + def _intern_stack(self, frame_id: int, prefix_id: Optional[int]) -> int: + """Gets a matching stack, or saves the new stack. Returns a Stack ID.""" + key = (prefix_id, frame_id) + stack_id = self.tables.stack_map.get(key) + if stack_id is None: + stack_id = len(self.tables.stack_table) + self.tables.stack_table.append(Stack(prefix_id=prefix_id, frame_id=frame_id)) + self.tables.stack_map[key] = stack_id + return stack_id + + def _intern_string(self, string: str) -> int: + """Gets a matching string, or saves the new string. Returns a String ID.""" + string_id = self.tables.string_map.get(string) + if string_id is not None: + return string_id + string_id = len(self.tables.string_table) + self.tables.string_table.append(string) + self.tables.string_map[string] = string_id + return string_id + + def _intern_frame(self, frame_str: str) -> int: + """Gets a matching stack frame, or saves the new frame. Returns a Frame ID.""" + frame_id = self.tables.frame_map.get(frame_str) + if frame_id is not None: + return frame_id + frame_id = len(self.tables.frame_table) + self.tables.frame_map[frame_str] = frame_id + string_id = self._intern_string(frame_str) + + category = self.user_category + if (frame_str.find('kallsyms') != -1 or + frame_str.find('/vmlinux') != -1 or + frame_str.endswith('.ko)')): + category = self.kernel_category + + self.tables.frame_table.append(Frame( + string_id=string_id, + relevantForJS=False, + innerWindowID=0, + implementation=None, + optimizations=None, + line=None, + column=None, + category=category, + subcategory=None, + )) + return frame_id + + def add_sample(self, comm: str, stack: List[str], time_ms: float) -> None: + """Add a timestamped stack trace sample to the thread builder.""" + if self.comm != comm: + self.comm = comm + + prefix_stack_id: Optional[int] = None + for frame in stack: + frame_id = self._intern_frame(frame) + prefix_stack_id = self._intern_stack(frame_id, prefix_stack_id) + + if prefix_stack_id is not None: + self.samples.append(Sample(stack_id=prefix_stack_id, + time_ms=time_ms, + responsiveness=0)) + + def to_json_dict(self) -> Dict: + """Converts current Thread to GeckoThread JSON format.""" + return { + "tid": self.tid, + "pid": self.pid, + "name": self.comm, + "markers": { + "schema": { + "name": 0, + "startTime": 1, + "endTime": 2, + "phase": 3, + "category": 4, + "data": 5, + }, + "data": [], + }, + "samples": { + "schema": { + "stack": 0, + "time": 1, + "responsiveness": 2, + }, + "data": self.samples + }, + "frameTable": { + "schema": { + "location": 0, + "relevantForJS": 1, + "innerWindowID": 2, + "implementation": 3, + "optimizations": 4, + "line": 5, + "column": 6, + "category": 7, + "subcategory": 8, + }, + "data": self.tables.frame_table, + }, + "stackTable": { + "schema": { + "prefix": 0, + "frame": 1, + }, + "data": self.tables.stack_table, + }, + "stringTable": self.tables.string_table, + "registerTime": 0, + "unregisterTime": None, + "processType": "default", + } + + +class CORSRequestHandler(SimpleHTTPRequestHandler): + """Enable CORS for requests from profiler.firefox.com.""" + def __init__(self, *args, allowed_file: Optional[str] = None, **kwargs): + self.allowed_file = allowed_file + super().__init__(*args, **kwargs) + + def end_headers(self): + self.send_header('Access-Control-Allow-Origin', 'https://profiler.firefox.com') + super().end_headers() + + def do_GET(self): + if self.allowed_file and self.path.split('?')[0] != f'/{self.allowed_file}': + self.send_error(403, "Access denied") + return + super().do_GET() + + def list_directory(self, path): + self.send_error(403, "Directory listing forbidden") + return None + + +@dataclass +class CategoryData: + """Category configuration for the gecko profile.""" + user_index: int = 0 + kernel_index: int = 1 + categories: List[Dict] = field(default_factory=list) + + +class GeckoCLI: + """Command-line interface for converting perf data to Gecko format.""" + def __init__(self, args: argparse.Namespace) -> None: + self.args = args + self.tid_to_thread: Dict[int, Thread] = {} + self.start_time_ms: Optional[float] = None + self.session: Optional[perf.session] = None + self.product = args.product + self.cat_data = CategoryData( + categories=[ + { + "name": 'User', + "color": args.user_color, + "subcategories": ['Other'] + }, + { + "name": 'Kernel', + "color": args.kernel_color, + "subcategories": ['Other'] + }, + ] + ) + + def process_event(self, sample) -> None: + """Process a single perf sample event.""" + if self.args.event_name and self.args.event_name not in str(sample.evsel): + return + + # sample_time is in nanoseconds. Gecko wants milliseconds. + time_ms = sample.sample_time / 1000000.0 + pid = sample.sample_pid + tid = sample.sample_tid + + if self.start_time_ms is None: + self.start_time_ms = time_ms + + try: + thread_info = self.session.find_thread(sample.sample_pid, sample.sample_tid) if self.session else None + comm = (thread_info.comm() if thread_info is not None else None) or "[unknown]" + except AttributeError: + comm = "[unknown]" + + stack = [] + callchain = sample.callchain + if callchain: + for entry in callchain: + symbol = entry.symbol or "[unknown]" + dso = entry.dso or "[unknown]" + stack.append(f"{symbol} (in {dso})") + # Reverse because Gecko wants root first. + stack.reverse() + else: + symbol = sample.symbol or "[unknown]" + dso = sample.dso or "[unknown]" + stack.append(f"{symbol} (in {dso})") + + thread = self.tid_to_thread.get(tid) + if thread is None: + thread = Thread(comm=comm, pid=pid, tid=tid, + user_category=self.cat_data.user_index, + kernel_category=self.cat_data.kernel_index) + self.tid_to_thread[tid] = thread + thread.add_sample(comm=comm, stack=stack, time_ms=time_ms) + + def run(self) -> None: + """Run the conversion process.""" + input_file = self.args.input or "perf.data" + if input_file != "-" and not os.path.exists(input_file): + print(f"Error: {input_file} not found.", file=sys.stderr) + sys.exit(1) + + try: + self.session = perf.session(perf.data(input_file), sample=self.process_event) + except (OSError, RuntimeError, ValueError) as e: + print(f"Error opening session: {e}", file=sys.stderr) + sys.exit(1) + + if self.session: + try: + self.session.process_events() + finally: + # Break the reference cycle between perf.session and self.process_event + # because perf.session lacks cyclic GC support (tp_traverse). + self.session = None + + threads = [t.to_json_dict() for t in self.tid_to_thread.values()] + + gecko_profile = { + "meta": { + "interval": 1, + "processType": 0, + "product": self.product, + "stackwalk": 1, + "debug": 0, + "gcpoison": 0, + "asyncstack": 1, + "startTime": self.start_time_ms or 0, + "shutdownTime": None, + "version": 24, + "presymbolicated": True, + "categories": self.cat_data.categories, + "markerSchema": [], + }, + "libs": [], + "threads": threads, + "processes": [], + "pausedRanges": [], + } + + output_file = self.args.save_only + if output_file is None: + self._write_and_launch(gecko_profile) + else: + print(f'[ perf gecko: Captured and wrote into {output_file} ]') + with open(output_file, 'w', encoding='utf-8') as f: + json.dump(gecko_profile, f, indent=2) + + def _write_and_launch(self, profile: Dict) -> None: + """Write the profile to a file and launch the Firefox profiler.""" + print("Starting Firefox Profiler on your default browser...") + + with tempfile.TemporaryDirectory() as tmp_dir_name: + profile_name = f'gecko_profile_{uuid.uuid4().hex}.json' + filename = os.path.join(tmp_dir_name, profile_name) + + with open(filename, 'w', encoding='utf-8') as f: + json.dump(profile, f, indent=2) + + handler = functools.partial( + CORSRequestHandler, directory=tmp_dir_name, allowed_file=profile_name + ) + try: + httpd = HTTPServer(('127.0.0.1', 0), handler) + except OSError as e: + print(f"Error starting HTTP server: {e}", file=sys.stderr) + sys.exit(1) + + port = httpd.server_port + + def start_server(): + httpd.serve_forever() + + thread = threading.Thread(target=start_server, daemon=True) + thread.start() + + safe_string = urllib.parse.quote_plus(f'http://127.0.0.1:{port}/{profile_name}') + url = f'https://profiler.firefox.com/from-url/{safe_string}' + print(f"Please open the following URL in your browser to view the profile:\n\n{url}\n") + launch_html = os.path.join(tmp_dir_name, 'launch.html') + with open(launch_html, 'w', encoding='utf-8') as f: + f.write(f'<!DOCTYPE html><html><head>' + f'<meta http-equiv="refresh" content="0;url={url}">' + f'<script>window.location.replace({json.dumps(url)});</script>' + f'</head><body>Redirecting to Firefox Profiler...</body></html>') + if not webbrowser.open(f'file://{launch_html}'): + print("Failed to open browser, please open the URL manually.") + + print(f'[ perf gecko: Captured and wrote into {filename} ]') + print("Press Ctrl+C to stop the local server.") + try: + # Keep the main thread alive so the daemon thread can serve requests + stop_event = threading.Event() + while True: + stop_event.wait(1) + except KeyboardInterrupt: + print("\nStopping server...") + httpd.shutdown() + + +if __name__ == "__main__": + parser = argparse.ArgumentParser( + description="Convert perf.data to Firefox's Gecko Profile format" + ) + parser.add_argument('--product', default='perf', help='Product name (e.g. perf)') + parser.add_argument('--user-color', default='yellow', + help='Color for the User category', + choices=['yellow', 'blue', 'purple', 'green', 'orange', 'red', + 'grey', 'magenta']) + parser.add_argument('--kernel-color', default='orange', + help='Color for the Kernel category', + choices=['yellow', 'blue', 'purple', 'green', 'orange', 'red', + 'grey', 'magenta']) + parser.add_argument('--save-only', + help='Save the output to a file instead of opening Firefox\'s profiler') + parser.add_argument("-i", "--input", help="input perf.data file") + parser.add_argument("-e", "--event", default="", dest="event_name", type=str, + help="specify the event to generate gecko profile for") + + cli_args = parser.parse_args() + cli = GeckoCLI(cli_args) + cli.run() diff --git a/tools/perf/python/ilist.py b/tools/perf/python/ilist.py index 0d757ddb4795..86bd65452e6e 100755 --- a/tools/perf/python/ilist.py +++ b/tools/perf/python/ilist.py @@ -2,11 +2,13 @@ # SPDX-License-Identifier: (LGPL-2.1 OR BSD-2-Clause) """Interactive perf list.""" -from abc import ABC, abstractmethod +from __future__ import annotations + import argparse +from abc import ABC, abstractmethod from dataclasses import dataclass import math -from typing import Any, Dict, Optional, Tuple +from typing import Any, Optional import perf from textual import on from textual.app import App, ComposeResult @@ -19,7 +21,7 @@ from textual.widgets import Button, Footer, Header, Input, Label, Sparkline, Sta from textual.widgets.tree import TreeNode -def get_info(info: Dict[str, str], key: str): +def get_info(info: dict[str, Any], key: str): return (info[key] + "\n") if key in info else "" @@ -80,7 +82,7 @@ class Metric(TreeValue): try: val = evlist.compute_metric(self.metric_name, cpu, thread) return 0 if math.isnan(val) else val - except: + except (OSError, ValueError, RuntimeError, TypeError): # Be tolerant of failures to compute metrics on particular CPUs/threads. return 0 @@ -140,7 +142,7 @@ class ErrorScreen(ModalScreen[bool]): def compose(self) -> ComposeResult: yield Button(f"Error: {self.error}", variant="primary", id="error") - def on_button_pressed(self, event: Button.Pressed) -> None: + def on_button_pressed(self) -> None: self.dismiss(True) @@ -234,7 +236,7 @@ class IListApp(App): def __init__(self, interval: float) -> None: self.interval = interval - self.evlist = None + self.evlist: Optional[perf.evlist] = None self.selected: Optional[TreeValue] = None self.search_results: list[TreeNode[TreeValue]] = [] self.cur_search_result: TreeNode[TreeValue] | None = None @@ -284,18 +286,18 @@ class IListApp(App): """Sets the focus after the SearchScreen is dismissed.""" search_label = self.query_one("#active_search", Label) - search_label.display = True if event else False + search_label.display = bool(event) if not event: return event = event.lower() search_label.update(f'Searching for events matching "{event}"') - tree: Tree[str] = self.query_one("#root", Tree) + tree: Tree[TreeValue] = self.query_one("#root", Tree) - def find_search_results(event: str, node: TreeNode[str], + def find_search_results(event: str, node: TreeNode[TreeValue], cursor_seen: bool = False, - match_after_cursor: Optional[TreeNode[str]] = None - ) -> Tuple[bool, Optional[TreeNode[str]]]: + match_after_cursor: Optional[TreeNode[TreeValue]] = None + ) -> tuple[bool, Optional[TreeNode[TreeValue]]]: """Find nodes that match the search remembering the one after the cursor.""" if not cursor_seen and node == tree.cursor_node: cursor_seen = True @@ -332,7 +334,7 @@ class IListApp(App): def action_collapse(self) -> None: """Collapse the part of the tree currently on.""" - tree: Tree[str] = self.query_one("#root", Tree) + tree: Tree[TreeValue] = self.query_one("#root", Tree) node = tree.cursor_node if node and node.parent: node.parent.collapse_all() @@ -343,31 +345,34 @@ class IListApp(App): if not self.selected or not self.evlist: return - def update_count(cpu: int, count: int): + def update_count(cpu: int, count: int | float): # Update the raw count display. - counter: Label = self.query(f"#counter_cpu{cpu}" if cpu >= 0 else "#counter_total") - if not counter: + counter_query = self.query(f"#counter_cpu{cpu}" if cpu >= 0 else "#counter_total") + if not counter_query: return - counter = counter.first(Label) + counter: Label = counter_query.first(Label) counter.update(str(count)) # Update the sparkline. - line: Sparkline = self.query(f"#sparkline_cpu{cpu}" if cpu >= 0 else "#sparkline_total") - if not line: + line_query = self.query(f"#sparkline_cpu{cpu}" if cpu >= 0 else "#sparkline_total") + if not line_query: return - line = line.first(Sparkline) + line: Sparkline = line_query.first(Sparkline) # If there are more events than the width, remove the front event. - if len(line.data) > line.size.width: - line.data.pop(0) - line.data.append(count) - line.mutate_reactive(Sparkline.data) + if line.data is not None: + if len(line.data) > line.size.width: + line.data = line.data[1:] + line.data = list(line.data) + [float(count)] + else: + line.data = [float(count)] + line.refresh() # Update the total and each CPU counts, assume there's just 1 evsel. - total = 0 + total: float = 0.0 self.evlist.disable() for evsel in self.evlist: for cpu in evsel.cpus(): - aggr = 0 + aggr: float = 0.0 for thread in evsel.threads(): aggr += self.selected.value(self.evlist, evsel, cpu, thread) update_count(cpu, aggr) @@ -414,7 +419,7 @@ class IListApp(App): if self.evlist: self.evlist.open() self.evlist.enable() - except: + except (OSError, ValueError, RuntimeError): self.evlist = None if not self.evlist: @@ -424,16 +429,16 @@ class IListApp(App): # Add spark lines for all the CPUs. Note, must be done after # open so that the evlist CPUs have been computed by propagate # maps. - line = CounterSparkline(cpu=-1) - lines.mount(line) + line_sp = CounterSparkline(cpu=-1) + lines.mount(line_sp) for cpu in self.evlist.all_cpus(): - line = CounterSparkline(cpu) - lines.mount(line) - line = Counter(cpu=-1) - lines.mount(line) + c_sp = CounterSparkline(cpu) + lines.mount(c_sp) + c_val = Counter(cpu=-1) + lines.mount(c_val) for cpu in self.evlist.all_cpus(): - line = Counter(cpu) - lines.mount(line) + c_val2 = Counter(cpu) + lines.mount(c_val2) def compose(self) -> ComposeResult: """Draws the app.""" @@ -445,7 +450,7 @@ class IListApp(App): pmu_name = pmu.name().lower() pmu_node = pmus.add(pmu_name) try: - for event in sorted(pmu.events(), key=lambda x: x["name"]): + for event in sorted(pmu.events(), key=lambda x: x.get("name", "")): if "deprecated" in event: continue if "name" in event: @@ -455,21 +460,22 @@ class IListApp(App): data=PmuEvent(pmu_name, e)) else: pmu_node.add_leaf(e, data=PmuEvent(pmu_name, e)) - except: + except (OSError, ValueError, RuntimeError, KeyError, TypeError): # Reading events may fail with EPERM, ignore. pass metrics = tree.root.add("Metrics") - groups = set() + groups: set[str] = set() for metric in perf.metrics(): groups.update(metric["MetricGroup"]) - def add_metrics_to_tree(node: TreeNode[TreeValue], parent: str, pmu: str = None): + def add_metrics_to_tree(node: TreeNode[TreeValue], parent: str, pmu: Optional[str] = None): for metric in sorted(perf.metrics(), key=lambda x: x["MetricName"]): - metric_pmu = metric.get('PMU') + metric_pmu_raw = metric.get('PMU') + metric_pmu = str(metric_pmu_raw) if metric_pmu_raw else '' if pmu and metric_pmu and metric_pmu != pmu: continue if parent in metric["MetricGroup"]: - name = metric["MetricName"] + name = str(metric["MetricName"]) display_name = name if metric_pmu: display_name += f" ({metric_pmu})" diff --git a/tools/perf/python/intel-pt-events.py b/tools/perf/python/intel-pt-events.py new file mode 100755 index 000000000000..c233a29eb8f9 --- /dev/null +++ b/tools/perf/python/intel-pt-events.py @@ -0,0 +1,632 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +""" +Print Intel PT Events including Power Events and PTWRITE. +Ported from tools/perf/scripts/python/intel-pt-events.py +""" +from __future__ import annotations + +import argparse +from typing import Dict, List, Tuple +import contextlib +from ctypes import addressof, create_string_buffer +import io +import os +import struct +import sys +from typing import Any, Optional +import typing +import perf + +try: + from libxed import LibXED as _LibXED + LibXED: Optional[type[_LibXED]] = _LibXED +except ImportError: + LibXED = None + + +def sample_flags_to_name(flags: int) -> str: + """Implement perf's sample_flags_to_name.""" + if not isinstance(flags, int) or flags == 0: + return "".ljust(21) + # PERF_IP_FLAG_* bit positions (from tools/perf/util/event.h): + # bit 0: PERF_IP_FLAG_BRANCH + # bit 1: PERF_IP_FLAG_CALL + # bit 2: PERF_IP_FLAG_RETURN + # bit 3: PERF_IP_FLAG_CONDITIONAL + # bit 4: PERF_IP_FLAG_SYSCALLRET + # bit 5: PERF_IP_FLAG_ASYNC + # bit 6: PERF_IP_FLAG_INTERRUPT + # bit 7: PERF_IP_FLAG_TX_ABORT + # bit 8: PERF_IP_FLAG_TRACE_BEGIN + # bit 9: PERF_IP_FLAG_TRACE_END + # bit 10: PERF_IP_FLAG_IN_TX + # bit 11: PERF_IP_FLAG_VMENTRY + # bit 12: PERF_IP_FLAG_VMEXIT + # bit 13: PERF_IP_FLAG_INTR_DISABLE + # bit 14: PERF_IP_FLAG_INTR_TOGGLE + # bit 15: PERF_IP_FLAG_BRANCH_MISS + # bit 16: PERF_IP_FLAG_NOT_TAKEN + sample_flags = [ + ((1 << 0) | (1 << 1), "call"), + ((1 << 0) | (1 << 2), "return"), + ((1 << 0) | (1 << 3), "jcc"), + ((1 << 0), "jmp"), + ((1 << 0) | (1 << 1) | (1 << 6), "int"), + ((1 << 0) | (1 << 2) | (1 << 6), "iret"), + ((1 << 0) | (1 << 1) | (1 << 4), "syscall"), + ((1 << 0) | (1 << 2) | (1 << 4), "sysret"), + ((1 << 0) | (1 << 5), "async"), + ((1 << 0) | (1 << 1) | (1 << 5) | (1 << 6), "hw int"), + ((1 << 0) | (1 << 7), "tx abrt"), + ((1 << 0) | (1 << 8), "tr strt"), + ((1 << 0) | (1 << 9), "tr end"), + ((1 << 0) | (1 << 1) | (1 << 11), "vmentry"), + ((1 << 0) | (1 << 1) | (1 << 12), "vmexit"), + ] + additional_mask = (1 << 10) | (1 << 13) | (1 << 14) + branch_event_mask = (1 << 15) | (1 << 16) + xf = flags & additional_mask + rem_flags = flags & ~additional_mask + + if rem_flags & (1 << 8): + prefix = "tr strt " + elif rem_flags & (1 << 9): + prefix = "tr end " + else: + prefix = "" + + rem_flags &= ~((1 << 8) | (1 << 9)) + types = rem_flags & ~branch_event_mask + type_name = "" + for f_mask, f_name in sample_flags: + if f_mask == types: + type_name = f_name + break + + s = prefix + type_name + ev_parts = [] + if rem_flags & (1 << 15): + ev_parts.append("miss") + if rem_flags & (1 << 16): + ev_parts.append("not_taken") + if ev_parts: + s += "/" + ",".join(ev_parts) + "/" + + if xf: + xs = "(" + if xf & (1 << 10): + xs += "x" + if xf & (1 << 13): + xs += "D" + if xf & (1 << 14): + xs += "t" + xs += ")" + if len(s) + len(xs) < 21: + s = s + xs.rjust(21 - len(s)) + else: + s = s + " " + xs + return s.ljust(21) + +class IntelPTAnalyzer: + """Analyzes Intel PT events and prints details.""" + + def __init__(self, cfg: argparse.Namespace): + self.args = cfg + self.session: Optional[perf.session] = None + self.insn = False + self.src = False + self.source_file_name: Optional[str] = None + self.line_number: int = 0 + self.dso: Optional[str] = None + self.stash_dict: Dict[int, List[str]] = {} + self.output: Any = None + self.output_pos: int = 0 + self.cpu: int = -1 + self.time: int = 0 + self.switch_str: Dict[int, str] = {} + + if cfg.insn_trace: + print("Intel PT Instruction Trace") + self.insn = True + elif cfg.src_trace: + print("Intel PT Source Trace") + self.insn = True + self.src = True + else: + print("Intel PT Branch Trace, Power Events, Event Trace and PTWRITE") + + self.disassembler: Any = None + self.inst: Any = None + if self.insn and LibXED is not None: + try: + self.disassembler = LibXED() + self.inst = self.disassembler.instruction() + except (OSError, ValueError, KeyError, RuntimeError, TypeError, AttributeError, + ImportError) as e: + print(f"Failed to initialize LibXED: {e}") + self.disassembler = None + self.inst = None + + def print_ptwrite(self, raw_buf: bytes) -> None: + """Print PTWRITE data.""" + if not raw_buf: + return + try: + data = struct.unpack_from("<IQ", raw_buf) + flags = data[0] + payload = data[1] + except struct.error: + return + exact_ip = flags & 1 + try: + s = payload.to_bytes(8, "little").decode("ascii").rstrip("\x00") + if not s.isprintable(): + s = "" + except (UnicodeDecodeError, ValueError): + s = "" + print(f"IP: {exact_ip} payload: {payload:#x} {s}", end=' ') + + def print_cbr(self, raw_buf: bytes) -> None: + """Print CBR data.""" + if len(raw_buf) < 12: + return + try: + data = struct.unpack_from("<BBBBII", raw_buf) + except struct.error: + return + cbr = data[0] + f = (data[4] + 500) // 1000 + if data[2] == 0: + return + p = ((cbr * 1000 // data[2]) + 5) // 10 + print(f"{cbr:3d} freq: {f:4d} MHz ({p:3d}%)", end=' ') + + def print_mwait(self, raw_buf: bytes) -> None: + """Print MWAIT data.""" + try: + data = struct.unpack_from("<IQ", raw_buf) + except struct.error: + return + payload = data[1] + hints = payload & 0xff + extensions = (payload >> 32) & 0x3 + print(f"hints: {hints:#x} extensions: {extensions:#x}", end=' ') + + def print_pwre(self, raw_buf: bytes) -> None: + """Print PWRE data.""" + try: + data = struct.unpack_from("<IQ", raw_buf) + except struct.error: + return + payload = data[1] + hw = (payload >> 7) & 1 + cstate = (payload >> 12) & 0xf + subcstate = (payload >> 8) & 0xf + print(f"hw: {hw} cstate: {cstate} sub-cstate: {subcstate}", end=' ') + + def print_exstop(self, raw_buf: bytes) -> None: + """Print EXSTOP data.""" + try: + data = struct.unpack_from("<I", raw_buf) + except struct.error: + return + flags = data[0] + exact_ip = flags & 1 + print(f"IP: {exact_ip}", end=' ') + + def print_pwrx(self, raw_buf: bytes) -> None: + """Print PWRX data.""" + try: + data = struct.unpack_from("<IQ", raw_buf) + except struct.error: + return + payload = data[1] + deepest_cstate = payload & 0xf + last_cstate = (payload >> 4) & 0xf + wake_reason = (payload >> 8) & 0xf + print(f"deepest cstate: {deepest_cstate} last cstate: {last_cstate} " + f"wake reason: {wake_reason:#x}", end=' ') + + def print_psb(self, raw_buf: bytes) -> None: + """Print PSB data.""" + try: + data = struct.unpack_from("<IQ", raw_buf) + except struct.error: + return + offset = data[1] + print(f"offset: {offset:#x}", end=' ') + + def print_evt(self, raw_buf: bytes) -> None: + """Print EVT data.""" + glb_cfe = ["", "INTR", "IRET", "SMI", "RSM", "SIPI", "INIT", "VMENTRY", "VMEXIT", + "VMEXIT_INTR", "SHUTDOWN", "", "UINT", "UIRET"] + [""] * 18 + glb_evd = ["", "PFA", "VMXQ", "VMXR"] + [""] * 60 + + try: + data = struct.unpack_from("<BBH", raw_buf) + except struct.error: + return + typ = data[0] & 0x1f + ip_flag = (data[0] & 0x80) >> 7 + vector = data[1] + evd_cnt = data[2] + s = glb_cfe[typ] + if s: + print(f" cfe: {s} IP: {ip_flag} vector: {vector}", end=' ') + else: + print(f" cfe: {typ} IP: {ip_flag} vector: {vector}", end=' ') + pos = 4 + for _ in range(evd_cnt): + if len(raw_buf) < pos + 16: + break + try: + data = struct.unpack_from("<QQ", raw_buf, pos) + except struct.error: + return + et = data[0] & 0x3f + s = glb_evd[et] + if s: + print(f"{s}: {data[1]:#x}", end=' ') + else: + print(f"EVD_{et}: {data[1]:#x}", end=' ') + pos += 16 + + def print_iflag(self, raw_buf: bytes) -> None: + """Print IFLAG data.""" + try: + data = struct.unpack_from("<IQ", raw_buf) + except struct.error: + return + iflag = data[0] & 1 + old_iflag = iflag ^ 1 + via_branch = data[0] & 2 + s = "via" if via_branch else "non" + print(f"IFLAG: {old_iflag}->{iflag} {s} branch", end=' ') + + def common_start_str(self, comm: str, sample: perf.sample_event) -> str: + """Return common start string for display.""" + ts = sample.sample_time + cpu = sample.sample_cpu + pid = sample.sample_pid + tid = sample.sample_tid + machine_pid = getattr(sample, "machine_pid", 0) + if machine_pid: + vcpu = getattr(sample, "vcpu", -1) + return (f"VM:{machine_pid:5d} VCPU:{vcpu:03d} {comm:>16s} {pid:5d}/{tid:<5d} " + f"[{cpu:03d}] {ts // 1000000000:9d}.{ts % 1000000000:09d} ") + return (f"{comm:>16s} {pid:5d}/{tid:<5d} [{cpu:03d}] " + f"{ts // 1000000000:9d}.{ts % 1000000000:09d} ") + + def print_common_start(self, comm: str, sample: perf.sample_event, name: str) -> None: + """Print common start info.""" + flags_disp = sample_flags_to_name(getattr(sample, "flags", 0)) + print(self.common_start_str(comm, sample) + f"{name:>8s} {flags_disp:>21s}", end=' ') + + def print_instructions_start(self, comm: str, sample: perf.sample_event) -> None: + """Print instructions start info.""" + raw_flags = getattr(sample, "flags", 0) + if isinstance(raw_flags, int) and (raw_flags & (1 << 10)): + print(self.common_start_str(comm, sample) + "x", end=' ') + else: + print(self.common_start_str(comm, sample), end=' ') + + def disassem(self, insn: bytes, ip: int) -> Tuple[int, str]: + """Disassemble instruction using LibXED.""" + inst = self.inst if self.inst is not None else self.disassembler.instruction() + is_64_bit = getattr(self.session, "is_64_bit", True) if self.session else True + self.disassembler.set_mode(inst, 0 if is_64_bit else 1) + buf = create_string_buffer(insn) + return self.disassembler.disassemble_one(inst, addressof(buf), len(insn), ip) + + def print_common_ip(self, sample: perf.sample_event, symbol: str, dso: str) -> None: + """Print IP and symbol info.""" + ip = sample.sample_ip + offs = f"+{sample.sym_offset:#x}" if getattr(sample, "sym_offset", None) is not None else "" + cyc_cnt = getattr(sample, "sample_cyc_count", 0) + if cyc_cnt: + insn_cnt = getattr(sample, "sample_insn_count", 0) + ipc_str = f" IPC: {insn_cnt / cyc_cnt:#.2f} ({insn_cnt}/{cyc_cnt})" + else: + ipc_str = "" + + if self.insn and self.disassembler is not None: + try: + insn = sample.insn() + except AttributeError: + insn = None + if insn: + cnt, text = self.disassem(insn, ip) + byte_str = (f"{ip:x}").rjust(16) + for k in range(cnt): + byte_str += f" {insn[k]:02x}" + print(f"{byte_str:<40s} {text:<30s}", end=' ') + else: + print(f"{ip:16x}", end=' ') + print(f"{symbol}{offs} ({dso})", end=' ') + else: + print(f"{ip:16x} {symbol}{offs} ({dso})", end=' ') + + addr = getattr(sample, "sample_addr", getattr(sample, "addr", 0)) + addr_correlates_sym = ( + bool(addr) + or getattr(sample, "addr_symbol", None) is not None + or getattr(sample, "addr_dso", None) is not None + ) + if addr_correlates_sym: + addr_dso = (sample.addr_dso or '[unknown]') + addr_symbol = (sample.addr_symbol or '[unknown]') + addr_offs = (f"+{sample.addr_sym_offset:#x}" + if getattr(sample, "addr_sym_offset", None) is not None else "") + print(f"=> {addr:x} {addr_symbol}{addr_offs} ({addr_dso}){ipc_str}") + else: + print(ipc_str) + + def print_srccode(self, comm: str, sample: perf.sample_event, + symbol: str, dso: str, with_insn: bool) -> None: + """Print source code info.""" + ip = sample.sample_ip + if symbol == "[unknown]": + start_str = self.common_start_str(comm, sample) + (f"{ip:x}").rjust(16).ljust(40) + else: + offs = (f"+{sample.sym_offset:#x}" + if getattr(sample, "sym_offset", None) is not None else "") + start_str = self.common_start_str(comm, sample) + (symbol + offs).ljust(40) + + if with_insn and self.insn and self.disassembler is not None: + try: + insn = sample.insn() + except AttributeError: + insn = None + if insn: + _, text = self.disassem(insn, ip) + start_str += text.ljust(30) + + source_file_name: typing.Any = None + line_number: typing.Any = 0 + source_line: typing.Any = None + try: + res_srcc = sample.srccode() + if res_srcc: + source_file_name, line_number, source_line = res_srcc + except (AttributeError, ValueError, TypeError): + pass + + if source_file_name: + if self.line_number == line_number and self.source_file_name == source_file_name: + src_str = "" + else: + if len(source_file_name) > 40: + src_file = ("..." + source_file_name[-37:]) + " " + else: + src_file = source_file_name.ljust(41) + if source_line is None: + src_str = src_file + str(line_number).rjust(4) + " <source not found>" + else: + src_str = src_file + str(line_number).rjust(4) + " " + source_line + self.dso = None + elif dso == self.dso: + src_str = "" + else: + src_str = dso + self.dso = dso + + self.line_number = line_number + self.source_file_name = source_file_name + print(start_str, src_str) + + def do_process_event(self, sample: perf.sample_event) -> None: + """Process event and print info.""" + cpu = getattr(sample, "sample_cpu", getattr(sample, "cpu", 0)) + if cpu in self.switch_str: + print(self.switch_str[cpu]) + del self.switch_str[cpu] + + comm = "Unknown" + if hasattr(self, 'session') and self.session: + try: + thread = self.session.find_thread(sample.sample_pid, sample.sample_tid) + if thread: + comm = thread.comm() or "Unknown" + except (OSError, ValueError, KeyError, RuntimeError, TypeError, AttributeError): + pass + # Python < 3.9 compatibility + name = str(sample.evsel) + if name.startswith("evsel("): + name = name[6:-1] + dso = (sample.dso or '[unknown]') + symbol = (sample.symbol or '[unknown]') + + raw_buf = getattr(sample, 'raw_buf', b'') or b'' + + if name.startswith("instructions"): + if self.src: + self.print_srccode(comm, sample, symbol, dso, True) + else: + self.print_instructions_start(comm, sample) + self.print_common_ip(sample, symbol, dso) + elif name.startswith("branches"): + if self.src: + self.print_srccode(comm, sample, symbol, dso, False) + else: + self.print_common_start(comm, sample, name) + self.print_common_ip(sample, symbol, dso) + elif name == "ptwrite": + self.print_common_start(comm, sample, name) + self.print_ptwrite(raw_buf) + self.print_common_ip(sample, symbol, dso) + elif name == "cbr": + self.print_common_start(comm, sample, name) + self.print_cbr(raw_buf) + self.print_common_ip(sample, symbol, dso) + elif name == "mwait": + self.print_common_start(comm, sample, name) + self.print_mwait(raw_buf) + self.print_common_ip(sample, symbol, dso) + elif name == "pwre": + self.print_common_start(comm, sample, name) + self.print_pwre(raw_buf) + self.print_common_ip(sample, symbol, dso) + elif name == "exstop": + self.print_common_start(comm, sample, name) + self.print_exstop(raw_buf) + self.print_common_ip(sample, symbol, dso) + elif name == "pwrx": + self.print_common_start(comm, sample, name) + self.print_pwrx(raw_buf) + self.print_common_ip(sample, symbol, dso) + elif name == "psb": + self.print_common_start(comm, sample, name) + self.print_psb(raw_buf) + self.print_common_ip(sample, symbol, dso) + elif name == "evt": + self.print_common_start(comm, sample, name) + self.print_evt(raw_buf) + self.print_common_ip(sample, symbol, dso) + elif name == "iflag": + self.print_common_start(comm, sample, name) + self.print_iflag(raw_buf) + self.print_common_ip(sample, symbol, dso) + else: + self.print_common_start(comm, sample, name) + self.print_common_ip(sample, symbol, dso) + + def interleave_events(self, sample: perf.sample_event) -> None: + """Interleave output to avoid garbled lines from different CPUs.""" + self.cpu = sample.sample_cpu + ts = sample.sample_time + + if self.time != ts: + self.time = ts + self.flush_stashed_output() + + self.output_pos = 0 + with contextlib.redirect_stdout(io.StringIO()) as self.output: + self.do_process_event(sample) + + self.stash_output() + + def stash_output(self) -> None: + """Stash output for later flushing.""" + output_str = self.output.getvalue()[self.output_pos:] + n = len(output_str) + if n: + self.output_pos += n + if self.cpu not in self.stash_dict: + self.stash_dict[self.cpu] = [] + self.stash_dict[self.cpu].append(output_str) + if len(self.stash_dict[self.cpu]) > 1000: + self.flush_stashed_output() + + def flush_stashed_output(self) -> None: + """Flush stashed output.""" + while self.stash_dict: + cpus = list(self.stash_dict.keys()) + for cpu in cpus: + items = self.stash_dict[cpu] + countdown = self.args.interleave + while len(items) and countdown: + sys.stdout.write(items[0]) + del items[0] + countdown -= 1 + if not items: + del self.stash_dict[cpu] + + def process_context_switch(self, event: perf.switch_event) -> None: + """Process context switch.""" + cpu = getattr(event, "sample_cpu", getattr(event, "cpu", 0)) + pid = getattr(event, "sample_pid", getattr(event, "pid", 0)) + tid = getattr(event, "sample_tid", getattr(event, "tid", 0)) + ts = getattr(event, "sample_time", getattr(event, "time", 0)) + np_pid = getattr(event, "next_prev_pid", None) + np_tid = getattr(event, "next_prev_tid", None) + machine_pid = getattr(event, "machine_pid", -1) + vcpu = getattr(event, "vcpu", -1) + misc = getattr(event, "misc", 0) + out = bool(misc & (1 << 13)) + out_preempt = bool(misc & (1 << 14)) + + if self.args.interleave: + self.flush_stashed_output() + + if out: + out_str = "Switch out " + else: + out_str = "Switch In " + + preempt_str = "preempt" if out_preempt else "" + + if machine_pid == -1: + machine_str = "" + elif vcpu == -1: + machine_str = f"machine PID {machine_pid}" + else: + machine_str = f"machine PID {machine_pid} VCPU {vcpu}" + + np_str = f"{np_pid:5d}/{np_tid:<5d} " if np_pid is not None and np_tid is not None else "" + c_str = (f"{out_str:>16s} {pid:5d}/{tid:<5d} [{cpu:03d}] " + f"{ts // 1000000000:9d}.{ts % 1000000000:09d} " + f"{np_str}{machine_str} {preempt_str}") + + if self.args.all_switch_events: + print(c_str) + else: + self.switch_str[cpu] = c_str + def process_event(self, sample: perf.sample_event) -> None: + """Wrapper to handle interleaving and exceptions.""" + try: + if self.args.interleave: + self.interleave_events(sample) + else: + self.do_process_event(sample) + except BrokenPipeError: + # Stop python printing broken pipe errors and traceback + sys.stdout = open(os.devnull, 'w', encoding='utf-8') + sys.exit(1) + + +if __name__ == "__main__": + ap = argparse.ArgumentParser() + ap.add_argument("-i", "--input", default="perf.data", help="Input file name") + ap.add_argument("--insn-trace", action='store_true') + ap.add_argument("--src-trace", action='store_true') + ap.add_argument("--all-switch-events", action='store_true') + ap.add_argument("--interleave", type=int, nargs='?', const=4, default=0) + ap.add_argument("--itrace", default=os.environ.get("PERF_ITRACE"), help="itrace options") + args = ap.parse_args() + + if args.itrace is None: + if args.insn_trace or args.src_trace: + args.itrace = "i0nsepwxI" + else: + args.itrace = "bepwxI" + analyzer = IntelPTAnalyzer(args) + + try: + # Note: Python API currently lacks auxtrace_error callbacks affecting + # chronological interleaving + try: + analyzer.session = perf.session( + perf.data(args.input), + sample=analyzer.process_event, + context_switch=analyzer.process_context_switch, + itrace=args.itrace + ) + try: + analyzer.session.process_events() + except KeyboardInterrupt: + pass + finally: + # Break the reference cycle between analyzer.session and the bound + # analyzer.process_event / process_context_switch callbacks so the + # underlying C perf.session object is freed. + analyzer.session = None + if args.interleave: + analyzer.flush_stashed_output() + print("End") + except BrokenPipeError: + sys.exit(0) + except (OSError, ValueError, KeyError, RuntimeError, TypeError, AttributeError): + import traceback + traceback.print_exc() + sys.exit(1) diff --git a/tools/perf/python/libxed.py b/tools/perf/python/libxed.py new file mode 100755 index 000000000000..900bd3e0184b --- /dev/null +++ b/tools/perf/python/libxed.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +""" +Python wrapper for libxed.so +Ported from tools/perf/scripts/python/libxed.py +""" +from __future__ import annotations + +from ctypes import CDLL, Structure, create_string_buffer, addressof, sizeof, \ + c_void_p, c_byte, c_int, c_uint, c_ulonglong + +# To use Intel XED, libxed.so must be present. To build and install +# libxed.so: +# git clone https://github.com/intelxed/mbuild.git mbuild +# git clone https://github.com/intelxed/xed +# cd xed +# ./mfile.py --share +# sudo ./mfile.py --prefix=/usr/local install +# sudo ldconfig +# + + +class XedStateT(Structure): + """xed_state_t structure.""" + _fields_ = [ + ("mode", c_int), + ("width", c_int) + ] + + +class XEDInstruction(): + """Represents a decoded instruction.""" + + def __init__(self, libxed): + # Current xed_decoded_inst_t structure is 192 bytes. Use 512 to allow for future expansion + xedd_t = c_byte * 512 + self.xedd = xedd_t() + self.xedp = addressof(self.xedd) + libxed.xed_decoded_inst_zero(self.xedp) + self.state = XedStateT() + self.statep = addressof(self.state) + # Buffer for disassembled instruction text + self.buffer = create_string_buffer(256) + self.bufferp = addressof(self.buffer) + + +class LibXED(): + """Wrapper for libxed.so.""" + + def __init__(self): + try: + self.libxed = CDLL("libxed.so") + except OSError: + self.libxed = None + if not self.libxed: + try: + self.libxed = CDLL("/usr/local/lib/libxed.so") + except OSError: + self.libxed = None + + if not self.libxed: + raise ImportError("libxed.so not found. Please install Intel XED.") + + self.xed_tables_init = self.libxed.xed_tables_init + self.xed_tables_init.restype = None + self.xed_tables_init.argtypes = [] + + self.xed_decoded_inst_zero = self.libxed.xed_decoded_inst_zero + self.xed_decoded_inst_zero.restype = None + self.xed_decoded_inst_zero.argtypes = [c_void_p] + + self.xed_operand_values_set_mode = self.libxed.xed_operand_values_set_mode + self.xed_operand_values_set_mode.restype = None + self.xed_operand_values_set_mode.argtypes = [c_void_p, c_void_p] + + self.xed_decoded_inst_zero_keep_mode = self.libxed.xed_decoded_inst_zero_keep_mode + self.xed_decoded_inst_zero_keep_mode.restype = None + self.xed_decoded_inst_zero_keep_mode.argtypes = [c_void_p] + + self.xed_decode = self.libxed.xed_decode + self.xed_decode.restype = c_int + self.xed_decode.argtypes = [c_void_p, c_void_p, c_uint] + + self.xed_format_context = self.libxed.xed_format_context + self.xed_format_context.restype = c_uint + self.xed_format_context.argtypes = [ + c_int, c_void_p, c_void_p, c_int, c_ulonglong, c_void_p, c_void_p + ] + + self.xed_decoded_inst_get_length = self.libxed.xed_decoded_inst_get_length + self.xed_decoded_inst_get_length.restype = c_uint + self.xed_decoded_inst_get_length.argtypes = [c_void_p] + + self.xed_tables_init() + + def instruction(self): + """Create a new XEDInstruction.""" + return XEDInstruction(self) + + def set_mode(self, inst, mode): + """Set 32-bit or 64-bit mode.""" + if mode: + inst.state.mode = 4 # 32-bit + inst.state.width = 4 # 4 bytes + else: + inst.state.mode = 1 # 64-bit + inst.state.width = 8 # 8 bytes + self.xed_operand_values_set_mode(inst.xedp, inst.statep) + + def disassemble_one(self, inst, bytes_ptr, bytes_cnt, ip): + """Disassemble one instruction.""" + self.xed_decoded_inst_zero_keep_mode(inst.xedp) + err = self.xed_decode(inst.xedp, bytes_ptr, bytes_cnt) + if err: + return 0, "" + # Use AT&T mode (2), alternative is Intel (3) + ok = self.xed_format_context(2, inst.xedp, inst.bufferp, sizeof(inst.buffer), ip, 0, 0) + if not ok: + return 0, "" + + result = inst.buffer.value.decode('utf-8') + # Return instruction length and the disassembled instruction text + return self.xed_decoded_inst_get_length(inst.xedp), result diff --git a/tools/perf/python/mem-phys-addr.py b/tools/perf/python/mem-phys-addr.py new file mode 100755 index 000000000000..53ef7e9a14b5 --- /dev/null +++ b/tools/perf/python/mem-phys-addr.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +"""mem-phys-addr.py: Resolve physical address samples""" +from __future__ import annotations +import argparse +import bisect +import collections +from dataclasses import dataclass +import re +from typing import (Dict, List, Optional) + +import perf + +@dataclass(frozen=True) +class IomemEntry: + """Read from a line in /proc/iomem""" + begin: int + end: int + indent: int + label: str + + def __lt__(self, other) -> bool: + if isinstance(other, int): + return self.begin < other + return self.begin < other.begin + + def __gt__(self, other) -> bool: + if isinstance(other, int): + return self.begin > other + return self.begin > other.begin + +# Physical memory layout from /proc/iomem. Key is the indent and then +# a list of ranges. +iomem: Dict[int, List[IomemEntry]] = collections.defaultdict(list) +# Child nodes from the iomem parent. +children: Dict[IomemEntry, List[IomemEntry]] = collections.defaultdict(list) +# Maximum indent seen before an entry in the iomem file. +_STATE: Dict[str, int] = {"max_indent": 0} +# Per-event counts for each range of memory. +event_counts: Dict[str, collections.Counter] = collections.defaultdict(collections.Counter) + +def parse_iomem(iomem_path: str): + """Populate iomem from iomem file""" + with open(iomem_path, 'r', encoding='ascii') as f: + for line in f: + line = line.rstrip('\n') + if not line or line.isspace(): + continue + indent = 0 + while indent < len(line) and line[indent] == ' ': + indent += 1 + _STATE["max_indent"] = max(_STATE["max_indent"], indent) + m = re.split('-|:', line, maxsplit=2) + if len(m) < 3: + continue + begin = int(m[0].strip(), 16) + end = int(m[1].strip(), 16) + label = m[2].strip() + entry = IomemEntry(begin, end, indent, label) + # Before adding entry, search for a parent node using its begin. + if indent > 0: + parent = find_memory_type(begin) + assert parent, f"Given indent expected a parent for {label}" + children[parent].append(entry) + iomem[indent].append(entry) + +def find_memory_type(phys_addr) -> Optional[IomemEntry]: + """Search iomem for the range containing phys_addr with the maximum indent""" + for i in range(_STATE["max_indent"], -1, -1): + if i not in iomem: + continue + position = bisect.bisect_right(iomem[i], phys_addr) + if position == 0: + continue + iomem_entry = iomem[i][position-1] + if iomem_entry.begin <= phys_addr <= iomem_entry.end: + return iomem_entry + return None + +def _print_entries(entries, load_mem_type_cnt, total): + """Print counts from parents down to their children""" + for entry in sorted(entries, + key=lambda e: (-load_mem_type_cnt[e], e.begin)): + count = load_mem_type_cnt[entry] + if count > 0: + mem_type = ' ' * entry.indent + f"{entry.begin:x}-{entry.end:x} : {entry.label}" + percent = 100 * count / total + print(f"{mem_type:<40} {count:>10} {percent:>10.1f}") + _print_entries(children[entry], load_mem_type_cnt, total) + +def print_memory_type(): + """Print the resolved memory types and their counts.""" + if not event_counts: + print("No valid physical address samples found in perf data.") + return + + for event_name, load_mem_type_cnt in event_counts.items(): + print(f"Event: {event_name}") + print(f"{'Memory type':<40} {'count':>10} {'percentage':>10}") + print(f"{'-' * 40:<40} {'-' * 10:>10} {'-' * 10:>10}") + total = sum(load_mem_type_cnt.values()) + if total == 0: + continue + + # Add count from children into the parent. + for i in range(_STATE["max_indent"], -1, -1): + if i not in iomem: + continue + for entry in iomem[i]: + for child in children[entry]: + if load_mem_type_cnt[child] > 0: + load_mem_type_cnt[entry] += load_mem_type_cnt[child] + + _print_entries(iomem[0], load_mem_type_cnt, total) + print() +if __name__ == "__main__": + ap = argparse.ArgumentParser(description="Resolve physical address samples") + ap.add_argument("-i", "--input", default="perf.data", help="Input file name") + ap.add_argument("--iomem", default="/proc/iomem", help="Path to iomem file") + args = ap.parse_args() + + def process_event(sample): + """Process a single sample event.""" + phys_addr = sample.sample_phys_addr or 0 + if not phys_addr: + return + entry = find_memory_type(phys_addr) + if entry: + event_name = str(sample.evsel) + if event_name.startswith("evsel(") and event_name.endswith(")"): + event_name = event_name[6:-1] + event_counts[event_name][entry] += 1 + + parse_iomem(args.iomem) + perf.session(perf.data(args.input), sample=process_event).process_events() + print_memory_type() diff --git a/tools/perf/python/net_dropmonitor.py b/tools/perf/python/net_dropmonitor.py new file mode 100755 index 000000000000..fff637b5c7e9 --- /dev/null +++ b/tools/perf/python/net_dropmonitor.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +""" +Monitor the system for dropped packets and produce a report of drop locations and counts. +Ported from tools/perf/scripts/python/net_dropmonitor.py +""" +from __future__ import annotations + +import argparse +from collections import defaultdict +import os +import sys +from typing import Tuple +import perf + + +class DropMonitor: + """Monitors dropped packets and aggregates counts by location.""" + + def __init__(self, kallsyms_path: str | None = None) -> None: + self.drop_log: dict[int, int] = defaultdict(int) + self.kallsyms: list[Tuple[int, str]] = [] + self.kallsyms_parsed = False + self.resolved_syms: dict[int, Tuple[str, int]] = {} + self.callchain_syms: dict[int, str] = {} + self.kallsyms_path = ( + kallsyms_path + or os.environ.get("PERF_SYMBOL_KALLSYMS") + or "/proc/kallsyms" + ) + + def _parse_kallsyms(self) -> None: + """Parse the kallsyms file and map kernel addresses to function symbols.""" + self.kallsyms.clear() + self.kallsyms_parsed = True + try: + with open(self.kallsyms_path, "r", encoding="utf-8") as f: + for line in f: + parts = line.split() + if len(parts) >= 3 and parts[1] in ('t', 'T', 'w', 'W'): + addr = int(parts[0], 16) + if addr > 0: + self.kallsyms.append((addr, parts[2])) + self.kallsyms.sort(key=lambda x: x[0]) + except (FileNotFoundError, PermissionError): + print(f"Failed to read {self.kallsyms_path}. Symbols will not be resolved.") + + def _get_sym(self, loc: int) -> Tuple[str, int]: + """Resolve a memory location using session symbols or the kallsyms map.""" + # Priority order: + # 1. Exact symbols with offsets resolved directly from the perf.data session + # (resolved_syms). + # 2. Symbols captured from the sample's callchain in the trace file + # (callchain_syms) before falling back to self.kallsyms, so offline + # trace symbols are not overwritten by the live host's /proc/kallsyms. + # 3. Binary search in self.kallsyms (from --kallsyms or /proc/kallsyms). + if loc in self.resolved_syms: + return self.resolved_syms[loc] + if loc in self.callchain_syms: + res = (self.callchain_syms[loc], 0) + self.resolved_syms[loc] = res + return res + if not self.kallsyms: + return f"{loc:#x}", 0 + + start = 0 + end = len(self.kallsyms) - 1 + while start < end: + mid = (start + end) // 2 + if self.kallsyms[mid][0] <= loc < self.kallsyms[mid+1][0]: + start = mid + break + if loc < self.kallsyms[mid][0]: + end = mid - 1 + else: + start = mid + 1 + + sym_addr, sym_name = self.kallsyms[start] + if loc >= sym_addr: + res = (sym_name, loc - sym_addr) + self.resolved_syms[loc] = res + return res + return f"{loc:#x}", 0 + + def print_drop_table(self) -> None: + """Print aggregated results.""" + if not self.drop_log: + print(f"{'LOCATION':>25} {'OFFSET':>25} {'COUNT':>25}") + return + + if (not self.kallsyms_parsed + and any(loc not in self.resolved_syms and loc not in self.callchain_syms + for loc in self.drop_log)): + print("Gathering kallsyms data") + self._parse_kallsyms() + + print(f"{'LOCATION':>25} {'OFFSET':>25} {'COUNT':>25}") + sorted_keys = sorted(self.drop_log.keys()) + for sloc in sorted_keys: + sym, off = self._get_sym(sloc) + print(f"{sym:>25} {off:>25d} {self.drop_log[sloc]:>25d}") + + def process_event(self, sample: perf.sample_event) -> None: + """Process a single sample event.""" + if "skb:kfree_skb" not in str(sample.evsel): + return + + location = getattr(sample, "location", None) + if location is not None: + self.drop_log[location] += 1 + if location not in self.resolved_syms: + sym = getattr(sample, "symbol", None) + if getattr(sample, "sample_ip", 0) == location and sym and sym != "[unknown]": + self.resolved_syms[location] = ( + sym, + getattr(sample, "sym_offset", 0) or 0, + ) + self.callchain_syms.pop(location, None) + else: + for entry in getattr(sample, "callchain", []) or []: + if isinstance(entry, dict): + entry_ip = entry.get("ip") + sym_info = entry.get("sym") + sym_name = sym_info.get("name") if isinstance(sym_info, dict) else None + sym_start = sym_info.get("start") if isinstance(sym_info, dict) else None + sym_off = (max(0, location - sym_start) + if sym_start is not None else None) + else: + entry_ip = getattr(entry, "ip", None) + entry_sym = getattr(entry, "sym", None) + sym_name = ( + getattr(entry_sym, "name", None) + or getattr(entry, "symbol", None) + ) + sym_off = getattr(entry, "sym_offset", None) + if sym_off is None: + sym_start = getattr(entry_sym, "start", None) + if sym_start is not None: + sym_off = max(0, location - sym_start) + if entry_ip == location and sym_name and sym_name != "[unknown]": + if sym_off is not None: + self.resolved_syms[location] = (sym_name, sym_off) + self.callchain_syms.pop(location, None) + else: + self.callchain_syms[location] = sym_name + break + + +if __name__ == "__main__": + ap = argparse.ArgumentParser( + description="Monitor the system for dropped packets and produce a " + "report of drop locations and counts.") + ap.add_argument("-i", "--input", default="perf.data", help="Input file name") + ap.add_argument("-k", "--kallsyms", default=None, + help="Path to kallsyms file for offline symbol resolution") + args = ap.parse_args() + + monitor = DropMonitor(kallsyms_path=args.kallsyms) + session = None + + try: + session = perf.session(perf.data(args.input), sample=monitor.process_event, + kallsyms=args.kallsyms) + session.process_events() + except KeyboardInterrupt: + print("\nStopping trace...") + except (OSError, ValueError, KeyError, RuntimeError, TypeError, AttributeError) as e: + print(f"Error processing events: {e}") + sys.exit(1) + finally: + session = None + + monitor.print_drop_table() diff --git a/tools/perf/python/netdev-times.py b/tools/perf/python/netdev-times.py new file mode 100755 index 000000000000..f0f829987447 --- /dev/null +++ b/tools/perf/python/netdev-times.py @@ -0,0 +1,494 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +""" +Display a process of packets and processed time. +It helps us to investigate networking or network device. + +Ported from tools/perf/scripts/python/netdev-times.py +""" + +from __future__ import annotations +import argparse +from collections import defaultdict +import sys +from typing import Optional +import perf + +# Format for displaying rx packet processing +PF_IRQ_ENTRY = " irq_entry(+%.3fmsec irq=%d:%s)" +PF_SOFT_ENTRY = " softirq_entry(+%.3fmsec)" +PF_NAPI_POLL = " napi_poll_exit(+%.3fmsec %s)" +PF_JOINT = " |" +PF_WJOINT = " | |" +PF_NET_RECV = " |---netif_receive_skb(+%.3fmsec skb=%x len=%d)" +PF_NET_RX = " |---netif_rx(+%.3fmsec skb=%x)" +PF_CPY_DGRAM = " | skb_copy_datagram_iovec(+%.3fmsec %d:%s)" +PF_KFREE_SKB = " | kfree_skb(+%.3fmsec location=%x)" +PF_CONS_SKB = " | consume_skb(+%.3fmsec)" + + +class NetDevTimesAnalyzer: + """Analyzes network device events and prints charts.""" + + def __init__(self, cfg: argparse.Namespace): + self.args = cfg + self.session: Optional[perf.session] = None + self.show_tx = cfg.tx or (not cfg.tx and not cfg.rx) + self.show_rx = cfg.rx or (not cfg.tx and not cfg.rx) + self.dev = cfg.dev + self.debug = cfg.debug + self.buffer_budget = 65536 + self.irq_dic: dict[int, list[dict]] = defaultdict(list) + self.net_rx_dic: dict[int, dict] = {} + self.receive_hunk_list: list[dict] = [] + self.rx_skb_list: list[dict] = [] + self.tx_queue_list: list[dict] = [] + self.tx_xmit_list: list[dict] = [] + self.tx_free_list: list[dict] = [] + + self.buffer_budget = 65536 + self.of_count_rx_skb_list = 0 + self.of_count_tx_queue_list = 0 + self.of_count_tx_xmit_list = 0 + + def diff_msec(self, src: int, dst: int) -> float: + """Calculate a time interval(msec) from src(nsec) to dst(nsec).""" + return (dst - src) / 1000000.0 + + def print_transmit(self, hunk: dict) -> None: + """Display a process of transmitting a packet.""" + if self.dev and hunk['dev'].find(self.dev) < 0: + return + queue_t_sec = hunk['queue_t'] // 1000000000 + queue_t_usec = hunk['queue_t'] % 1000000000 // 1000 + print(f"{hunk['dev']:7s} {hunk['len']:5d} " + f"{queue_t_sec:6d}.{queue_t_usec:06d}sec " + f"{self.diff_msec(hunk['queue_t'], hunk['xmit_t']):12.3f}msec " + f"{self.diff_msec(hunk['xmit_t'], hunk['free_t']):12.3f}msec") + + def print_receive(self, hunk: dict) -> None: + """Display a process of received packets and interrupts.""" + show_hunk = False + irq_list = hunk['irq_list'] + if not irq_list: + return + cpu = irq_list[0]['cpu'] + base_t = irq_list[0]['irq_ent_t'] + + if self.dev: + for irq in irq_list: + if irq['name'].find(self.dev) >= 0: + show_hunk = True + break + else: + show_hunk = True + + if not show_hunk: + return + + base_t_sec = base_t // 1000000000 + base_t_usec = base_t % 1000000000 // 1000 + print(f"{base_t_sec}.{base_t_usec:06d}sec cpu={cpu}") + for irq in irq_list: + print(PF_IRQ_ENTRY % + (self.diff_msec(base_t, irq['irq_ent_t']), + irq['irq'], irq['name'])) + print(PF_JOINT) + irq_event_list = irq['event_list'] + for irq_event in irq_event_list: + if irq_event['event'] == 'netif_rx': + print(PF_NET_RX % + (self.diff_msec(base_t, irq_event['time']), + irq_event['skbaddr'])) + print(PF_JOINT) + + print(PF_SOFT_ENTRY % self.diff_msec(base_t, hunk['sirq_ent_t'])) + print(PF_JOINT) + event_list = hunk['event_list'] + for i, event in enumerate(event_list): + if event['event_name'] == 'napi_poll': + print(PF_NAPI_POLL % + (self.diff_msec(base_t, event['event_t']), + event['dev'])) + if i == len(event_list) - 1: + print("") + else: + print(PF_JOINT) + else: + print(PF_NET_RECV % + (self.diff_msec(base_t, event['event_t']), + event['skbaddr'], + event['len'])) + if 'handle' in event: + print(PF_WJOINT) + if event['handle'] == "kfree_skb": + print(PF_KFREE_SKB % + (self.diff_msec(base_t, event['comm_t']), + event['location'])) + elif event['handle'] == "consume_skb": + print(PF_CONS_SKB % + self.diff_msec(base_t, event['comm_t'])) + elif event['handle'] == "skb_copy_datagram_iovec": + print(PF_CPY_DGRAM % + (self.diff_msec(base_t, event['comm_t']), + event['pid'], event['comm'])) + print(PF_JOINT) + + def handle_irq_handler_entry(self, event: dict) -> None: + """Handle irq:irq_handler_entry event.""" + time = event['time'] + cpu = event['cpu'] + irq = event['irq'] + irq_name = event['irq_name'] + irq_record = {'irq': irq, 'name': irq_name, 'cpu': cpu, + 'irq_ent_t': time, 'event_list': []} + self.irq_dic[cpu].append(irq_record) + + def handle_irq_handler_exit(self, event: dict) -> None: + """Handle irq:irq_handler_exit event.""" + time = event['time'] + cpu = event['cpu'] + irq = event['irq'] + if cpu not in self.irq_dic or not self.irq_dic[cpu]: + return + if irq != self.irq_dic[cpu][-1]['irq']: + return + irq_record = self.irq_dic[cpu].pop() + irq_record['irq_ext_t'] = time + # if an irq doesn't include NET_RX softirq, drop. + if irq_record['event_list']: + self.irq_dic[cpu].append(irq_record) + + def handle_irq_softirq_raise(self, event: dict) -> None: + """Handle irq:softirq_raise event.""" + time = event['time'] + cpu = event['cpu'] + if cpu not in self.irq_dic or not self.irq_dic[cpu]: + return + irq_record = self.irq_dic[cpu].pop() + irq_record['event_list'].append({'time': time, 'event': 'sirq_raise'}) + self.irq_dic[cpu].append(irq_record) + + def handle_irq_softirq_entry(self, event: dict) -> None: + """Handle irq:softirq_entry event.""" + time = event['time'] + cpu = event['cpu'] + self.net_rx_dic[cpu] = {'sirq_ent_t': time, 'event_list': []} + + def handle_irq_softirq_exit(self, event: dict) -> None: + """Handle irq:softirq_exit event.""" + time = event['time'] + cpu = event['cpu'] + irq_list = None + event_list = None + sirq_ent_t = None + + if cpu in self.irq_dic: + irq_list = self.irq_dic[cpu] + del self.irq_dic[cpu] + if cpu in self.net_rx_dic: + sirq_ent_t = self.net_rx_dic[cpu]['sirq_ent_t'] + event_list = self.net_rx_dic[cpu]['event_list'] + del self.net_rx_dic[cpu] + if not irq_list or event_list is None or sirq_ent_t is None: + return + rec_data = {'sirq_ent_t': sirq_ent_t, 'sirq_ext_t': time, + 'irq_list': irq_list, 'event_list': event_list} + self.receive_hunk_list.append(rec_data) + + def handle_napi_poll(self, event: dict) -> None: + """Handle napi:napi_poll event.""" + time = event['time'] + cpu = event['cpu'] + dev_name = event['dev_name'] + work = event['work'] + budget = event['budget'] + if cpu in self.net_rx_dic: + event_list = self.net_rx_dic[cpu]['event_list'] + rec_data = {'event_name': 'napi_poll', + 'dev': dev_name, 'event_t': time, + 'work': work, 'budget': budget} + event_list.append(rec_data) + + def handle_netif_rx(self, event: dict) -> None: + """Handle net:netif_rx event.""" + time = event['time'] + cpu = event['cpu'] + skbaddr = event['skbaddr'] + skblen = event['skblen'] + dev_name = event['dev_name'] + if cpu not in self.irq_dic or not self.irq_dic[cpu]: + return + irq_record = self.irq_dic[cpu].pop() + irq_record['event_list'].append({'time': time, 'event': 'netif_rx', + 'skbaddr': skbaddr, 'skblen': skblen, + 'dev_name': dev_name}) + self.irq_dic[cpu].append(irq_record) + + def handle_netif_receive_skb(self, event: dict) -> None: + """Handle net:netif_receive_skb event.""" + time = event['time'] + cpu = event['cpu'] + skbaddr = event['skbaddr'] + skblen = event['skblen'] + if cpu in self.net_rx_dic: + rec_data = {'event_name': 'netif_receive_skb', + 'event_t': time, 'skbaddr': skbaddr, 'len': skblen} + event_list = self.net_rx_dic[cpu]['event_list'] + event_list.append(rec_data) + self.rx_skb_list.insert(0, rec_data) + if len(self.rx_skb_list) > self.buffer_budget: + self.rx_skb_list.pop() + self.of_count_rx_skb_list += 1 + + def handle_net_dev_queue(self, event: dict) -> None: + """Handle net:net_dev_queue event.""" + time = event['time'] + skbaddr = event['skbaddr'] + skblen = event['skblen'] + dev_name = event['dev_name'] + skb = {'dev': dev_name, 'skbaddr': skbaddr, 'len': skblen, 'queue_t': time} + self.tx_queue_list.insert(0, skb) + if len(self.tx_queue_list) > self.buffer_budget: + self.tx_queue_list.pop() + self.of_count_tx_queue_list += 1 + + def handle_net_dev_xmit(self, event: dict) -> None: + """Handle net:net_dev_xmit event.""" + time = event['time'] + skbaddr = event['skbaddr'] + rc = event['rc'] + if rc == 0: # NETDEV_TX_OK + for i, skb in enumerate(self.tx_queue_list): + if skb['skbaddr'] == skbaddr: + skb['xmit_t'] = time + self.tx_xmit_list.insert(0, skb) + del self.tx_queue_list[i] + if len(self.tx_xmit_list) > self.buffer_budget: + self.tx_xmit_list.pop() + self.of_count_tx_xmit_list += 1 + return + + def handle_kfree_skb(self, event: dict) -> None: + """Handle skb:kfree_skb event.""" + time = event['time'] + skbaddr = event['skbaddr'] + comm = event['comm'] + pid = event['pid'] + location = event['location'] + for i, skb in enumerate(self.tx_queue_list): + if skb['skbaddr'] == skbaddr: + del self.tx_queue_list[i] + return + for i, skb in enumerate(self.tx_xmit_list): + if skb['skbaddr'] == skbaddr: + skb['free_t'] = time + self.tx_free_list.append(skb) + del self.tx_xmit_list[i] + return + for i, rec_data in enumerate(self.rx_skb_list): + if rec_data['skbaddr'] == skbaddr: + rec_data.update({'handle': "kfree_skb", + 'comm': comm, 'pid': pid, 'comm_t': time, 'location': location}) + del self.rx_skb_list[i] + return + + def handle_consume_skb(self, event: dict) -> None: + """Handle skb:consume_skb event.""" + time = event['time'] + skbaddr = event['skbaddr'] + for i, skb in enumerate(self.tx_xmit_list): + if skb['skbaddr'] == skbaddr: + skb['free_t'] = time + self.tx_free_list.append(skb) + del self.tx_xmit_list[i] + return + for i, rec_data in enumerate(self.rx_skb_list): + if rec_data['skbaddr'] == skbaddr: + rec_data.update({'handle': "consume_skb", 'comm_t': time}) + del self.rx_skb_list[i] + return + + def handle_skb_copy_datagram_iovec(self, event: dict) -> None: + """Handle skb:skb_copy_datagram_iovec event.""" + time = event['time'] + skbaddr = event['skbaddr'] + comm = event['comm'] + pid = event['pid'] + for i, rec_data in enumerate(self.rx_skb_list): + if skbaddr == rec_data['skbaddr']: + rec_data.update({'handle': "skb_copy_datagram_iovec", + 'comm': comm, 'pid': pid, 'comm_t': time}) + del self.rx_skb_list[i] + return + + def print_summary(self) -> None: + """Print charts.""" + + # display receive hunks + if self.show_rx: + for hunk in self.receive_hunk_list: + self.print_receive(hunk) + + # display transmit hunks + if self.show_tx: + print(" dev len Qdisc " + " netdevice free") + for hunk in self.tx_free_list: + self.print_transmit(hunk) + + if self.debug: + print("debug buffer status") + print("----------------------------") + print(f"xmit Qdisc:remain:{len(self.tx_queue_list)} " + f"overflow:{self.of_count_tx_queue_list}") + print(f"xmit netdevice:remain:{len(self.tx_xmit_list)} " + f"overflow:{self.of_count_tx_xmit_list}") + print(f"receive:remain:{len(self.rx_skb_list)} " + f"overflow:{self.of_count_rx_skb_list}") + + def handle_single_event(self, event: dict) -> None: + """Handle a single processed event.""" + name = event['name'] + if name == 'irq:softirq_exit': + self.handle_irq_softirq_exit(event) + elif name == 'irq:softirq_entry': + self.handle_irq_softirq_entry(event) + elif name == 'irq:softirq_raise': + self.handle_irq_softirq_raise(event) + elif name == 'irq:irq_handler_entry': + self.handle_irq_handler_entry(event) + elif name == 'irq:irq_handler_exit': + self.handle_irq_handler_exit(event) + elif name == 'napi:napi_poll': + self.handle_napi_poll(event) + elif name == 'net:netif_receive_skb': + self.handle_netif_receive_skb(event) + elif name == 'net:netif_rx': + self.handle_netif_rx(event) + elif name == 'skb:skb_copy_datagram_iovec': + self.handle_skb_copy_datagram_iovec(event) + elif name == 'net:net_dev_queue': + self.handle_net_dev_queue(event) + elif name == 'net:net_dev_xmit': + self.handle_net_dev_xmit(event) + elif name == 'skb:kfree_skb': + self.handle_kfree_skb(event) + elif name == 'skb:consume_skb': + self.handle_consume_skb(event) + + def process_event(self, sample: perf.sample_event) -> None: + """Process events directly on-the-fly.""" + name = str(sample.evsel) + ev_name = name[6:-1] if name.startswith("evsel(") else name + pid = sample.sample_pid + if hasattr(self, 'session') and self.session: + try: + thread = self.session.find_thread(sample.sample_pid, sample.sample_tid) + comm = (thread.comm() if thread else None) or "[unknown]" + except (OSError, ValueError, KeyError, RuntimeError, TypeError, AttributeError): + comm = "[unknown]" + else: + comm = "Unknown" + event_data = { + 'name': ev_name, + 'time': sample.sample_time, + 'cpu': sample.sample_cpu, + 'pid': pid, + 'comm': comm, + } + + # Extract specific fields based on event type + if ev_name.startswith("irq:softirq_"): + event_data['vec'] = getattr(sample, "vec", 0) + # Filter for NET_RX + if event_data['vec'] != 3: # NET_RX_SOFTIRQ is usually 3 + return + elif ev_name == "irq:irq_handler_entry": + event_data['irq'] = getattr(sample, "irq", -1) + event_data['irq_name'] = getattr(sample, "name", "[unknown]") + elif ev_name == "irq:irq_handler_exit": + event_data['irq'] = getattr(sample, "irq", -1) + event_data['ret'] = getattr(sample, "ret", 0) + elif ev_name == "napi:napi_poll": + event_data['napi'] = getattr(sample, "napi", 0) + event_data['dev_name'] = getattr(sample, "dev_name", "[unknown]") + event_data['work'] = getattr(sample, "work", 0) + event_data['budget'] = getattr(sample, "budget", 0) + elif ev_name in ("net:netif_receive_skb", "net:netif_rx", + "net:net_dev_queue"): + event_data['skbaddr'] = getattr(sample, "skbaddr", 0) + event_data['skblen'] = getattr(sample, "len", 0) + event_data['dev_name'] = getattr(sample, "name", "[unknown]") + elif ev_name == "net:net_dev_xmit": + event_data['skbaddr'] = getattr(sample, "skbaddr", 0) + event_data['skblen'] = getattr(sample, "len", 0) + event_data['rc'] = getattr(sample, "rc", 0) + event_data['dev_name'] = getattr(sample, "name", "[unknown]") + elif ev_name == "skb:kfree_skb": + event_data['skbaddr'] = getattr(sample, "skbaddr", 0) + event_data['location'] = getattr(sample, "location", 0) + event_data['protocol'] = getattr(sample, "protocol", 0) + event_data['reason'] = getattr(sample, "reason", 0) + elif ev_name == "skb:consume_skb": + event_data['skbaddr'] = getattr(sample, "skbaddr", 0) + event_data['location'] = getattr(sample, "location", 0) + elif ev_name == "skb:skb_copy_datagram_iovec": + event_data['skbaddr'] = getattr(sample, "skbaddr", 0) + event_data['skblen'] = getattr(sample, "len", 0) + + self.handle_single_event(event_data) + + +NetdevTimes = NetDevTimesAnalyzer + + +if __name__ == "__main__": + ap = argparse.ArgumentParser(description="Display a process of packets and processed time.") + ap.add_argument("-i", "--input", default="perf.data", help="Input file name") + ap.add_argument("--tx", action="store_true", help="show only tx chart") + ap.add_argument("--rx", action="store_true", help="show only rx chart") + ap.add_argument("--dev", default=None, help="show only specified device") + ap.add_argument("--debug", action="store_true", + help="work with debug mode. It shows buffer status.") + ap.add_argument("positionals", nargs="*", + help="optional positional arguments (tx, rx, dev=<name>, debug)") + args, unknown_args = ap.parse_known_args() + + parsed_args = argparse.Namespace( + tx=args.tx, rx=args.rx, dev=args.dev, debug=args.debug, input=args.input + ) + + for arg in list(args.positionals) + unknown_args: + if not arg or not isinstance(arg, str): + continue + if arg in ('tx', '--tx'): + parsed_args.tx = True + elif arg in ('rx', '--rx'): + parsed_args.rx = True + elif arg.startswith('dev='): + parsed_args.dev = arg[4:] + elif arg.startswith('--dev='): + parsed_args.dev = arg[6:] + elif arg in ('debug', '--debug'): + parsed_args.debug = True + + analyzer = NetDevTimesAnalyzer(parsed_args) + + try: + try: + analyzer.session = perf.session( + perf.data(parsed_args.input), sample=analyzer.process_event + ) + try: + analyzer.session.process_events() + except KeyboardInterrupt: + pass + finally: + # Break the reference cycle between analyzer.session and the bound + # analyzer.process_event callback so the C perf.session object is freed. + analyzer.session = None + analyzer.print_summary() + except (OSError, ValueError, KeyError, RuntimeError, TypeError, AttributeError) as e: + print(f"Error processing events: {e}") + sys.exit(1) diff --git a/tools/perf/python/parallel-perf.py b/tools/perf/python/parallel-perf.py new file mode 100755 index 000000000000..a43b003634fd --- /dev/null +++ b/tools/perf/python/parallel-perf.py @@ -0,0 +1,1250 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +"""Parallel perf script.""" +# +# run a perf script command multiple times in parallel, using perf script +# options --cpu and --time so that each job processes a different chunk +# of the data. +# +# Copyright (c) 2024, Intel Corporation. + +import subprocess +import argparse +import pathlib +import shlex +import time +from typing import Any +import copy +import sys +import os +import re + +glb_prog_name = "parallel-perf.py" +glb_min_interval = 10.0 +glb_min_samples = 64 + + +class Verbosity(): + + def __init__(self, quiet: bool = False, verbose: bool = False, + debug: bool = False) -> None: + """__init__.""" + + self.normal = True + self.verbose = verbose + self.debug = debug + self.self_test = True + if self.debug: + self.verbose = True + if self.verbose: + quiet = False + if quiet: + self.normal = False + +# Manage work (start/wait/kill), as represented by a subprocess.Popen command + + +class Work(): + + def __init__(self, cmd: list[str], pipe_to: str, + output_dir: str = ".") -> None: + """__init__.""" + + self.popen: Any = None + self.consumer: Any = None + self.cmd = cmd + self.pipe_to = pipe_to + self.output_dir = output_dir + self.cmdout_name = f"{output_dir}/cmd.txt" + self.stdout_name = f"{output_dir}/out.txt" + self.stderr_name = f"{output_dir}/err.txt" + + def command(self): + """command.""" + + return " ".join(shlex.quote(x) for x in self.cmd) + + def stdout(self): + """stdout.""" + + return open(self.stdout_name, "w", encoding="utf-8") + + def stderr(self): + """stderr.""" + + return open(self.stderr_name, "w", encoding="utf-8") + + def create_output_dir(self): + """create_output_dir.""" + + pathlib.Path(self.output_dir).mkdir(parents=True, exist_ok=True) + + def start(self): + """start.""" + + if self.popen: + return + self.create_output_dir() + with open(self.cmdout_name, "w", encoding="utf-8") as f: + f.write(self.command()) + f.write("\n") + stdout = self.stdout() + stderr = self.stderr() + if self.pipe_to: + self.popen = subprocess.Popen( + self.cmd, stdout=subprocess.PIPE, stderr=stderr) + args = shlex.split(self.pipe_to) + self.consumer = subprocess.Popen( + args, stdin=self.popen.stdout, stdout=stdout, stderr=stderr) + # The consumer now owns the read end of the pipe. Close this + # process's copy so that the producer receives SIGPIPE if the + # consumer exits early, and so that the descriptor isn't leaked + # for the lifetime of this Work. + self.popen.stdout.close() + self.popen.stdout = None + else: + self.popen = subprocess.Popen( + self.cmd, stdout=stdout, stderr=stderr) + + def remove_empty_err_file(self): + """remove_empty_err_file.""" + + if os.path.exists(self.stderr_name): + if os.path.getsize(self.stderr_name) == 0: + os.unlink(self.stderr_name) + + def errors(self): + """errors.""" + + if os.path.exists(self.stderr_name): + if os.path.getsize(self.stderr_name) != 0: + return [f"Non-empty error file {self.stderr_name}"] + return [] + + def tidy_up(self): + """tidy_up.""" + + self.remove_empty_err_file() + + def raw_poll_wait(self, p, wait): + """raw_poll_wait.""" + + if wait: + return p.wait() + return p.poll() + + def poll(self, wait=False): + """poll.""" + + if not self.popen: + return None + result = self.raw_poll_wait(self.popen, wait) + if self.consumer: + res = result + result = self.raw_poll_wait(self.consumer, wait) + if result is not None and res is None: + self.popen.kill() + result = None + elif result == 0 and res is not None and res != 0: + result = res + if result is not None: + self.tidy_up() + return result + + def wait(self): + """wait.""" + + return self.poll(wait=True) + + def kill(self): + """kill.""" + + if not self.popen: + return + self.popen.kill() + if self.consumer: + self.consumer.kill() + + +def kill_work(worklist, _verbosity): + """kill_work.""" + + for w in worklist: + w.kill() + for w in worklist: + w.wait() + + +def number_of_cp_us(): + """number_of_cp_us.""" + + return os.sysconf("SC_NPROCESSORS_ONLN") + + +def nano_secs_to_secs_str(x): + """nano_secs_to_secs_str.""" + + if x is None: + return "" + x = str(x) + if len(x) < 10: + x = "0" * (10 - len(x)) + x + return x[:len(x) - 9] + "." + x[-9:] + + +def insert_option_after(cmd, option, after): + """insert_option_after.""" + + try: + pos = cmd.index(after) + cmd.insert(pos + 1, option) + except (OSError, ValueError, RuntimeError): + cmd.append(option) + + +def create_work_list(cmd, pipe_to, output_dir, cpus, time_ranges_by_cpu): + """create_work_list.""" + + max_len = len(str(cpus[-1])) + cpu_dir_fmt = f"cpu-%.{max_len}u" + worklist = [] + pos = 0 + for cpu in cpus: + if cpu >= 0: + cpu_dir = os.path.join(output_dir, cpu_dir_fmt % cpu) + cpu_option = f"--cpu={cpu}" + else: + cpu_dir = output_dir + cpu_option = None + + tr_dir_fmt = "time-range" + + if len(time_ranges_by_cpu) > 1: + time_ranges = time_ranges_by_cpu[pos] + tr_dir_fmt += f"-{pos}" + pos += 1 + else: + time_ranges = time_ranges_by_cpu[0] + + max_len = len(str(len(time_ranges))) + tr_dir_fmt += f"-%.{max_len}u" + + i = 0 + for r in time_ranges: + if r == [None, None]: + time_option = None + work_output_dir = cpu_dir + else: + time_option = "--time=" + \ + nano_secs_to_secs_str(r[0]) + "," + \ + nano_secs_to_secs_str(r[1]) + work_output_dir = os.path.join(cpu_dir, tr_dir_fmt % i) + i += 1 + work_cmd = list(cmd) + if time_option is not None: + insert_option_after(work_cmd, time_option, "script") + if cpu_option is not None: + insert_option_after(work_cmd, cpu_option, "script") + w = Work(work_cmd, pipe_to, work_output_dir) + worklist.append(w) + return worklist + + +def do_run_work(worklist: list[Work], nr_jobs: int, + _verbosity: Verbosity) -> bool: + """do_run_work.""" + + nr_to_do = len(worklist) + not_started = list(worklist) + running: list[Work] = [] + done: list[Work] = [] + chg = False + while True: + nr_done = len(done) + if chg and _verbosity.normal: + nr_run = len(running) + print( + f"\rThere are {nr_to_do} jobs: {nr_done} completed, {nr_run} running", + flush=True, end=" ") + if _verbosity.verbose: + print() + chg = False + if nr_done == nr_to_do: + break + while len(running) < nr_jobs and len(not_started): + w = not_started.pop(0) + running.append(w) + if _verbosity.verbose: + print("Starting:", w.command()) + w.start() + chg = True + if len(running): + time.sleep(0.1) + finished: list[Work] = [] + not_finished: list[Work] = [] + while len(running): + w = running.pop(0) + r = w.poll() + if r is None: + not_finished.append(w) + continue + if r == 0: + if _verbosity.verbose: + print("Finished:", w.command()) + finished.append(w) + chg = True + continue + if _verbosity.normal and not _verbosity.verbose: + print() + print("Job failed!\n return code:", r, + "\n command: ", w.command()) + if w.pipe_to: + print(" piped to: ", w.pipe_to) + print("Killing outstanding jobs") + kill_work(not_finished, _verbosity) + kill_work(running, _verbosity) + return False + running = not_finished + done += finished + errorlist: list[str] = [] + for w in worklist: + errorlist += w.errors() + if len(errorlist): + print("errors:") + for e in errorlist: + print(e) + elif _verbosity.normal: + print("\r", " "*50, "\rAll jobs finished successfully", flush=True) + return True + + +def run_work(worklist: list[Work], nr_jobs: int = number_of_cp_us(), + _verbosity: Verbosity = Verbosity()) -> bool: + """run_work.""" + try: + return do_run_work(worklist, nr_jobs, _verbosity) + except BaseException: + for w in worklist: + w.kill() + raise + + +def read_header(perf, file_name): + """read_header.""" + + cmd = [perf, "script", "--header-only", "--input", file_name] + with subprocess.Popen(cmd, stdout=subprocess.PIPE) as proc: + out = proc.stdout.read() if proc.stdout else b"" + return out.decode("utf-8") + + +def parse_header(hdr): + """parse_header.""" + + result = {} + lines = hdr.split("\n") + for line in lines: + if ":" in line and line[0] == "#": + pos = line.index(":") + name = line[1:pos-1].strip() + value = line[pos+1:].strip() + if name in result: + orig_name = name + nr = 2 + while True: + name = f"{orig_name} {nr}" + if name not in result: + break + nr += 1 + result[name] = value + return result + + +def header_field(hdr_dict, hdr_fld): + """header_field.""" + + if hdr_fld not in hdr_dict: + raise RuntimeError(f"'{hdr_fld}' missing from header information") + return hdr_dict[hdr_fld] + +# Represent the position of an option within a command string +# and provide the option value and/or remove the option + + +class OptPos(): + + def init(self, opt_element=-1, value_element=-1, opt_pos=-1, value_pos=-1, error=None): + """init.""" + + self.opt_element = opt_element # list element that contains option + self.value_element = value_element # list element that contains option value + self.opt_pos = opt_pos # string position of option + self.value_pos = value_pos # string position of value + self.error = error # error message string + + def __init__(self, args, short_name, long_name, default=None): + """__init__.""" + + self.args = list(args) + self.default = default + n = 2 + len(long_name) + m = len(short_name) + pos = -1 + for opt in args: + pos += 1 + if m and opt[:2] == f"-{short_name}": + if len(opt) == 2: + if pos + 1 < len(args): + self.init(pos, pos + 1, 0, 0) + else: + self.init(error=f"-{short_name} option missing value") + else: + self.init(pos, pos, 0, 2) + return + if opt[:n] == f"--{long_name}": + if len(opt) == n: + if pos + 1 < len(args): + self.init(pos, pos + 1, 0, 0) + else: + self.init(error=f"--{long_name} option missing value") + elif opt[n] == "=": + self.init(pos, pos, 0, n + 1) + else: + self.init(error=f"--{long_name} option expected '='") + return + if m and opt[:1] == "-" and opt[:2] != "--" and short_name in opt: + ipos = opt.index(short_name) + if "-" in opt[1:]: + hpos = opt[1:].index("-") + if hpos < ipos: + continue + if ipos + 1 == len(opt): + if pos + 1 < len(args): + self.init(pos, pos + 1, ipos, 0) + else: + self.init(error=f"-{short_name} option missing value") + else: + self.init(pos, pos, ipos, ipos + 1) + return + self.init() + + def value(self): + """value.""" + + if self.opt_element >= 0: + if self.opt_element != self.value_element: + return self.args[self.value_element] + else: + return self.args[self.value_element][self.value_pos:] + return self.default + + def remove(self, args): + """remove.""" + + if self.opt_element == -1: + return + if self.opt_element != self.value_element: + del args[self.value_element] + if self.opt_pos: + args[self.opt_element] = args[self.opt_element][:self.opt_pos] + else: + del args[self.opt_element] + + +def determine_input_file_name(cmd): + """determine_input_file_name.""" + + p = OptPos(cmd, "i", "input", "perf.data") + if p.error: + raise RuntimeError(f"perf command {p.error}") + file_name = p.value() + if not os.path.exists(file_name): + raise RuntimeError(f"perf command input file '{file_name}' not found") + return file_name + + +def read_option(args, short_name, long_name, err_prefix, remove=False): + """read_option.""" + + p = OptPos(args, short_name, long_name) + if p.error: + raise RuntimeError(f"{err_prefix}{p.error}") + value = p.value() + if remove: + p.remove(args) + return value + + +def extract_option(args, short_name, long_name, err_prefix): + """extract_option.""" + + return read_option(args, short_name, long_name, err_prefix, True) + + +def read_perf_option(args, short_name, long_name): + """read_perf_option.""" + + return read_option(args, short_name, long_name, "perf command ") + + +def extract_perf_option(args, short_name, long_name): + """extract_perf_option.""" + + return extract_option(args, short_name, long_name, "perf command ") + + +def perf_double_quick_commands(cmd, file_name): + """perf_double_quick_commands.""" + + cpu_str = read_perf_option(cmd, "C", "cpu") + time_str = read_perf_option(cmd, "", "time") + # Use double-quick sampling to determine trace data density + times_cmd = ["perf", "script", "--ns", + "--input", file_name, "--itrace=qqi"] + if cpu_str is not None and cpu_str != "": + times_cmd.append(f"--cpu={cpu_str}") + if time_str is not None and time_str != "": + times_cmd.append(f"--time={time_str}") + cnts_cmd = list(times_cmd) + cnts_cmd.append("-Fcpu") + times_cmd.append("-Fcpu,time") + return cnts_cmd, times_cmd + + +class CPUTimeRange(): + def __init__(self, cpu): + """__init__.""" + + self.cpu = cpu + self.sample_cnt = 0 + self.time_ranges = None + self.interval = 0 + self.interval_remaining = 0 + self.remaining = 0 + self.tr_pos = 0 + + +def calc_time_ranges_by_cpu(line, cpu, cpu_time_ranges, max_time): + """calc_time_ranges_by_cpu.""" + + cpu_time_range = cpu_time_ranges[cpu] + cpu_time_range.remaining -= 1 + cpu_time_range.interval_remaining -= 1 + if cpu_time_range.remaining == 0: + cpu_time_range.time_ranges[cpu_time_range.tr_pos][1] = max_time + return + if cpu_time_range.interval_remaining == 0: + ts = time_val(line[1][:-1], 0) + time_ranges = cpu_time_range.time_ranges + time_ranges[cpu_time_range.tr_pos][1] = ts - 1 + time_ranges.append([ts, max_time]) + cpu_time_range.tr_pos += 1 + cpu_time_range.interval_remaining = cpu_time_range.interval + + +def count_samples_by_cpu(_line: list[str], cpu: int, + cpu_time_ranges: list[Any]) -> None: + """count_samples_by_cpu.""" + + try: + cpu_time_ranges[cpu].sample_cnt += 1 + except (OSError, ValueError, RuntimeError, IndexError): + print("exception") + print("cpu", cpu) + print("len(cpu_time_ranges)", len(cpu_time_ranges)) + raise + + +def process_command_output_lines(cmd, per_cpu, fn, *x): + """process_command_output_lines.""" + + # Assume CPU number is at beginning of line and enclosed by [] + pat = re.compile(r"\s*\[[0-9]+\]") + p = subprocess.Popen(cmd, stdout=subprocess.PIPE) + while True: + line = p.stdout.readline() + if line: + line = line.decode("utf-8") + if pat.match(line): + line = line.split() + if per_cpu: + # Assumes CPU number is enclosed by [] + cpu = int(line[0][1:-1]) + else: + cpu = 0 + fn(line, cpu, *x) + else: + break + p.wait() + + +def intersect_time_ranges(new_time_ranges, time_ranges): + """intersect_time_ranges.""" + + pos = 0 + new_pos = 0 + # Can assume len(time_ranges) != 0 and len(new_time_ranges) != 0 + # Note also, there *must* be at least one intersection. + while pos < len(time_ranges) and new_pos < len(new_time_ranges): + # new end < old start => no intersection, remove new + if new_time_ranges[new_pos][1] < time_ranges[pos][0]: + del new_time_ranges[new_pos] + continue + # new start > old end => no intersection, check next + if new_time_ranges[new_pos][0] > time_ranges[pos][1]: + pos += 1 + if pos < len(time_ranges): + continue + # no next, so remove remaining + while new_pos < len(new_time_ranges): + del new_time_ranges[new_pos] + return + # Found an intersection + # new start < old start => adjust new start = old start + if new_time_ranges[new_pos][0] < time_ranges[pos][0]: + new_time_ranges[new_pos][0] = time_ranges[pos][0] + # new end > old end => keep the overlap, insert the remainder + if new_time_ranges[new_pos][1] > time_ranges[pos][1]: + r = [time_ranges[pos][1] + 1, new_time_ranges[new_pos][1]] + new_time_ranges[new_pos][1] = time_ranges[pos][1] + new_pos += 1 + new_time_ranges.insert(new_pos, r) + continue + # new [start, end] is within old [start, end] + new_pos += 1 + + +def split_time_ranges_by_trace_data_density( + time_ranges, cpus, nr, cmd, file_name, per_cpu, min_size, min_interval, _verbosity +): + """split_time_ranges_by_trace_data_density.""" + + if _verbosity.normal: + print("\rAnalyzing...", flush=True, end=" ") + if _verbosity.verbose: + print() + cnts_cmd, times_cmd = perf_double_quick_commands(cmd, file_name) + + nr_cpus = cpus[-1] + 1 if per_cpu else 1 + if per_cpu: + nr_cpus = cpus[-1] + 1 + cpu_time_ranges = [CPUTimeRange(cpu) for cpu in range(nr_cpus)] + else: + nr_cpus = 1 + cpu_time_ranges = [CPUTimeRange(-1)] + + if _verbosity.debug: + print("nr_cpus", nr_cpus) + print("cnts_cmd", cnts_cmd) + print("times_cmd", times_cmd) + + # Count the number of "double quick" samples per CPU + process_command_output_lines( + cnts_cmd, per_cpu, count_samples_by_cpu, cpu_time_ranges) + + tot = 0 + mx = 0 + for cpu_time_range in cpu_time_ranges: + cnt = cpu_time_range.sample_cnt + tot += cnt + if cnt > mx: + mx = cnt + if _verbosity.debug: + print("cpu:", cpu_time_range.cpu, "sample_cnt", cnt) + + if min_size < 1: + min_size = 1 + + if mx < min_size: + # Too little data to be worth splitting + if _verbosity.debug: + print("Too little data to split by time") + if nr == 0: + nr = 1 + return [split_time_ranges_into_n(time_ranges, nr, min_interval)] + + if nr: + divisor = nr + min_size = 1 + else: + divisor = number_of_cp_us() + + interval = int(round(tot / divisor, 0)) + if interval < min_size: + interval = min_size + + if _verbosity.debug: + print("divisor", divisor) + print("min_size", min_size) + print("interval", interval) + + min_time = time_ranges[0][0] + max_time = time_ranges[-1][1] + + for cpu_time_range in cpu_time_ranges: + cnt = cpu_time_range.sample_cnt + if cnt == 0: + cpu_time_range.time_ranges = copy.deepcopy(time_ranges) + continue + # Adjust target interval for CPU to give approximately equal interval sizes + # Determine number of intervals, rounding to nearest integer + n = int(round(cnt / interval, 0)) + if n < 1: + n = 1 + # Determine interval size, rounding up + d, m = divmod(cnt, n) + if m: + d += 1 + cpu_time_range.interval = d + cpu_time_range.interval_remaining = d + cpu_time_range.remaining = cnt + # init. time ranges for each CPU with the start time + cpu_time_range.time_ranges = [[min_time, max_time]] + + # Set time ranges so that the same number of "double quick" samples + # will fall into each time range. + process_command_output_lines( + times_cmd, per_cpu, calc_time_ranges_by_cpu, cpu_time_ranges, max_time) + + for cpu_time_range in cpu_time_ranges: + if cpu_time_range.sample_cnt: + intersect_time_ranges(cpu_time_range.time_ranges, time_ranges) + + return [cpu_time_ranges[cpu].time_ranges for cpu in cpus] + + +def split_single_time_range_into_n(time_range, n): + """split_single_time_range_into_n.""" + + if n <= 1: + return [time_range] + start = time_range[0] + end = time_range[1] + duration = int((end - start + 1) / n) + if duration < 1: + return [time_range] + time_ranges = [] + for _i in range(n): + time_ranges.append([start, start + duration - 1]) + start += duration + time_ranges[-1][1] = end + return time_ranges + + +def time_range_duration(r): + """time_range_duration.""" + + return r[1] - r[0] + 1 + + +def total_duration(time_ranges): + """total_duration.""" + + duration = 0 + for r in time_ranges: + duration += time_range_duration(r) + return duration + + +def split_time_ranges_by_interval(time_ranges, interval): + """split_time_ranges_by_interval.""" + + new_ranges = [] + for r in time_ranges: + duration = time_range_duration(r) + n = duration / interval + n = int(round(n, 0)) + new_ranges += split_single_time_range_into_n(r, n) + return new_ranges + + +def split_time_ranges_into_n(time_ranges, n, min_interval): + """split_time_ranges_into_n.""" + + if n <= len(time_ranges): + return time_ranges + duration = total_duration(time_ranges) + interval = duration / n + if interval < min_interval: + interval = min_interval + return split_time_ranges_by_interval(time_ranges, interval) + + +def recombine_time_ranges(tr): + """recombine_time_ranges.""" + + new_tr = copy.deepcopy(tr) + i = 1 + while i < len(new_tr): + # if prev end + 1 == cur start, combine them + if new_tr[i - 1][1] + 1 == new_tr[i][0]: + new_tr[i][0] = new_tr[i - 1][0] + del new_tr[i - 1] + else: + i += 1 + return new_tr + + +def open_time_range_ends(time_ranges, min_time, max_time): + """open_time_range_ends.""" + + if time_ranges[0][0] <= min_time: + time_ranges[0][0] = None + if time_ranges[-1][1] >= max_time: + time_ranges[-1][1] = None + + +def bad_time_str(time_str): + """bad_time_str.""" + + raise RuntimeError( + f"perf command bad time option: '{time_str}'\n" + "Check also 'time of first sample' and 'time of last sample' " + "in perf script --header-only" + ) + + +def validate_time_ranges(time_ranges, time_str): + """validate_time_ranges.""" + + n = len(time_ranges) + for i in range(n): + start = time_ranges[i][0] + end = time_ranges[i][1] + if i != 0 and start <= time_ranges[i - 1][1]: + bad_time_str(time_str) + if start > end: + bad_time_str(time_str) + + +def time_val(s, dflt): + """time_val.""" + + s = s.strip() + if s == "": + return dflt + a = s.split(".") + if len(a) > 2: + raise RuntimeError(f"Bad time value'{s}'") + x = int(a[0]) + if x < 0: + raise RuntimeError("Negative time not allowed") + x *= 1000000000 + if len(a) > 1: + x += int((a[1] + "000000000")[:9]) + return x + + +def bad_cpu_str(cpu_str): + """bad_cpu_str.""" + + raise RuntimeError( + f"perf command bad cpu option: '{cpu_str}'\n" + "Check also 'nrcpus avail' in perf script --header-only" + ) + + +def parse_time_str(time_str, min_time, max_time): + """parse_time_str.""" + + if time_str is None or time_str == "": + return [[min_time, max_time]] + time_ranges = [] + for r in time_str.split(): + a = r.split(",") + if len(a) != 2: + bad_time_str(time_str) + try: + start = time_val(a[0], min_time) + end = time_val(a[1], max_time) + except (OSError, ValueError, RuntimeError): + bad_time_str(time_str) + time_ranges.append([start, end]) + validate_time_ranges(time_ranges, time_str) + return time_ranges + + +def parse_cpu_str(cpu_str, nr_cpus): + """parse_cpu_str.""" + + if cpu_str is None or cpu_str == "": + return [-1] + cpus = [] + for r in cpu_str.split(","): + a = r.split("-") + if len(a) < 1 or len(a) > 2: + bad_cpu_str(cpu_str) + try: + start = int(a[0].strip()) + if len(a) > 1: + end = int(a[1].strip()) + else: + end = start + except (OSError, ValueError, RuntimeError): + bad_cpu_str(cpu_str) + if start < 0 or end < 0 or end < start or end >= nr_cpus: + bad_cpu_str(cpu_str) + cpus.extend(range(start, end + 1)) + cpus = list(set(cpus)) # remove duplicates + cpus.sort() + return cpus + + +class ParallelPerf(): + + def __init__(self, a): + """init.""" + self.nr = 0 + self.jobs = 0 + self.file_name = None + self.hdr = None + self.hdr_dict = None + self.cmd_line = None + self.min_time = None + self.max_time = None + self.time_str = None + self.time_ranges = None + self.cpu_str = None + self.cpus = None + self.split_time_ranges_for_each_cpu = None + self.worklist = None + self.per_cpu = None + self.interval = None + self.min_size = None + self.min_interval = None + self.pipe_to = None + self.output_dir = None + self.cmd = getattr(a, 'cmd', None) + self.no_per_cpu = getattr(a, 'no_per_cpu', False) + self.dry_run = getattr(a, 'dry_run', False) + self._verbosity = getattr(a, "_verbosity", Verbosity()) + + for arg_name in vars(a): + setattr(self, arg_name, getattr(a, arg_name)) + self.orig_nr = self.nr + self.orig_cmd = list(self.cmd) + self.perf = self.cmd[0] + if os.path.exists(self.output_dir): + raise RuntimeError(f"Output '{self.output_dir}' already exists") + if self.jobs < 0 or self.nr < 0 or self.interval < 0: + raise RuntimeError( + "Bad options (negative values): try -h option for help") + if self.nr != 0 and self.interval != 0: + raise RuntimeError( + "Cannot specify number of time subdivisions and time interval") + if self.jobs == 0: + self.jobs = number_of_cp_us() + if self.nr == 0 and self.interval == 0: + if self.per_cpu: + self.nr = 1 + else: + self.nr = self.jobs + + def init(self): + """init.""" + + if self._verbosity.debug: + print("cmd", self.cmd) + self.file_name = determine_input_file_name(self.cmd) + self.hdr = read_header(self.perf, self.file_name) + self.hdr_dict = parse_header(self.hdr) + self.cmd_line = header_field(self.hdr_dict, "cmdline") + + def extract_time_info(self): + """extract_time_info.""" + + self.min_time = time_val(header_field( + self.hdr_dict, "time of first sample"), 0) + self.max_time = time_val(header_field( + self.hdr_dict, "time of last sample"), 0) + self.time_str = extract_perf_option(self.cmd, "", "time") + self.time_ranges = parse_time_str( + self.time_str, self.min_time, self.max_time) + if self._verbosity.debug: + print("time_ranges", self.time_ranges) + + def extract_cpu_info(self): + """extract_cpu_info.""" + + if self.per_cpu: + nr_cpus = int(header_field(self.hdr_dict, "nrcpus avail")) + self.cpu_str = extract_perf_option(self.cmd, "C", "cpu") + if self.cpu_str is None or self.cpu_str == "": + self.cpus = [x for x in range(nr_cpus)] + else: + self.cpus = parse_cpu_str(self.cpu_str, nr_cpus) + else: + self.cpu_str = None + self.cpus = [-1] + if self._verbosity.debug: + print("cpus", self.cpus) + + def is_intel_pt(self): + """is_intel_pt.""" + + return self.cmd_line.find("intel_pt") >= 0 + + def split_time_ranges(self): + """split_time_ranges.""" + + if self.is_intel_pt() and self.interval == 0: + self.split_time_ranges_for_each_cpu = ( + split_time_ranges_by_trace_data_density( + self.time_ranges, self.cpus, self.orig_nr, + self.orig_cmd, self.file_name, self.per_cpu, + self.min_size, self.min_interval, self._verbosity + ) + ) + elif self.nr: + self.split_time_ranges_for_each_cpu = [split_time_ranges_into_n( + self.time_ranges, self.nr, self.min_interval)] + else: + self.split_time_ranges_for_each_cpu = [ + split_time_ranges_by_interval(self.time_ranges, self.interval)] + + def check_time_ranges(self): + """check_time_ranges.""" + + for tr in self.split_time_ranges_for_each_cpu: + # Re-combined time ranges should be the same + new_tr = recombine_time_ranges(tr) + if new_tr != self.time_ranges: + if self._verbosity.debug: + print("tr", tr) + print("new_tr", new_tr) + raise RuntimeError("Self test failed!") + + def open_time_range_ends(self): + """open_time_range_ends.""" + + for time_ranges in self.split_time_ranges_for_each_cpu: + open_time_range_ends(time_ranges, self.min_time, self.max_time) + + def create_work_list(self): + """create_work_list.""" + + self.worklist = create_work_list( + self.cmd, self.pipe_to, self.output_dir, self.cpus, self.split_time_ranges_for_each_cpu) + + def perf_data_recorded_per_cpu(self): + """perf_data_recorded_per_cpu.""" + + if "--per-thread" in self.cmd_line.split(): + return False + return True + + def default_to_per_cpu(self): + """default_to_per_cpu.""" + + # --no-per-cpu option takes precedence + if self.no_per_cpu: + return False + if not self.perf_data_recorded_per_cpu(): + return False + # Default to per-cpu for Intel PT data that was recorded per-cpu, + # because decoding can be done for each CPU separately. + if self.is_intel_pt(): + return True + return False + + def config(self): + """config.""" + + self.init() + self.extract_time_info() + if not self.per_cpu: + self.per_cpu = self.default_to_per_cpu() + if self._verbosity.debug: + print("per_cpu", self.per_cpu) + self.extract_cpu_info() + self.split_time_ranges() + if self._verbosity.self_test: + self.check_time_ranges() + # Prefer open-ended time range to starting / ending with min_time / max_time resp. + self.open_time_range_ends() + self.create_work_list() + + def run(self): + """run.""" + + if self.dry_run: + print(len(self.worklist), "jobs:") + for w in self.worklist: + print(w.command()) + return True + result = run_work(self.worklist, self.jobs, _verbosity=self._verbosity) + if self._verbosity.verbose: + print(glb_prog_name, "done") + return result + + +def run_parallel_perf(a): + """run_parallel_perf.""" + + pp = ParallelPerf(a) + pp.config() + return pp.run() + + +def main(args): + """main.""" + + ap = argparse.ArgumentParser( + prog=glb_prog_name, formatter_class=argparse.RawDescriptionHelpFormatter, + description=""" +run a perf script command multiple times in parallel, using perf script options +--cpu and --time so that each job processes a different chunk of the data. +""", + epilog=""" +Follow the options by '--' and then the perf script command e.g. + + $ perf record -a -- sleep 10 + $ parallel-perf.py --nr=4 -- perf script --ns + All jobs finished successfully + $ tree parallel-perf-output/ + parallel-perf-output/ + ├── time-range-0 + │ ├── cmd.txt + │ └── out.txt + ├── time-range-1 + │ ├── cmd.txt + │ └── out.txt + ├── time-range-2 + │ ├── cmd.txt + │ └── out.txt + └── time-range-3 + ├── cmd.txt + └── out.txt + $ find parallel-perf-output -name cmd.txt | sort | xargs grep -H . + parallel-perf-output/time-range-0/cmd.txt:perf script --time=,9466.504461499 --ns + parallel-perf-output/time-range-1/cmd.txt:perf script --time=9466.504461500,9469.005396999 --ns + parallel-perf-output/time-range-2/cmd.txt:perf script --time=9469.005397000,9471.506332499 --ns + parallel-perf-output/time-range-3/cmd.txt:perf script --time=9471.506332500, --ns + +Any perf script command can be used, including the use of perf script options +--dlfilter and --script, so that the benefit of running parallel jobs +naturally extends to them also. + +If option --pipe-to is used, standard output is first piped through that +command. Beware, if the command fails (e.g. grep with no matches), it will be +considered a fatal error. + +Final standard output is redirected to files named out.txt in separate +subdirectories under the output directory. Similarly, standard error is +written to files named err.txt. In addition, files named cmd.txt contain the +corresponding perf script command. After processing, err.txt files are removed +if they are empty. + +If any job exits with a non-zero exit code, then all jobs are killed and no +more are started. A message is printed if any job results in a non-empty +err.txt file. + +There is a separate output subdirectory for each time range. If the --per-cpu +option is used, these are further grouped under cpu-n subdirectories, e.g. + + $ parallel-perf.py --per-cpu --nr=2 -- perf script --ns --cpu=0,1 + All jobs finished successfully + $ tree parallel-perf-output + parallel-perf-output/ + ├── cpu-0 + │ ├── time-range-0 + │ │ ├── cmd.txt + │ │ └── out.txt + │ └── time-range-1 + │ ├── cmd.txt + │ └── out.txt + └── cpu-1 + ├── time-range-0 + │ ├── cmd.txt + │ └── out.txt + └── time-range-1 + ├── cmd.txt + └── out.txt + $ find parallel-perf-output -name cmd.txt | sort | xargs grep -H . + parallel-perf-output/cpu-0/time-range-0/cmd.txt:perf script --cpu=0 --time=,9469.005396999 --ns + parallel-perf-output/cpu-0/time-range-1/cmd.txt:perf script --cpu=0 --time=9469.005397000, --ns + parallel-perf-output/cpu-1/time-range-0/cmd.txt:perf script --cpu=1 --time=,9469.005396999 --ns + parallel-perf-output/cpu-1/time-range-1/cmd.txt:perf script --cpu=1 --time=9469.005397000, --ns + +Subdivisions of time range, and cpus if the --per-cpu option is used, are +expressed by the --time and --cpu perf script options respectively. If the +supplied perf script command has a --time option, then that time range is +subdivided, otherwise the time range given by 'time of first sample' to +'time of last sample' is used (refer perf script --header-only). Similarly, the +supplied perf script command may provide a --cpu option, and only those CPUs +will be processed. + +To prevent time intervals becoming too small, the --min-interval option can +be used. + +Note there is special handling for processing Intel PT traces. If an interval is +not specified and the perf record command contained the intel_pt event, then the +time range will be subdivided in order to produce subdivisions that contain +approximately the same amount of trace data. That is accomplished by counting +double-quick (--itrace=qqi) samples, and choosing time ranges that encompass +approximately the same number of samples. In that case, time ranges may not be +the same for each CPU processed. For Intel PT, --per-cpu is the default, but +that can be overridden by --no-per-cpu. Note, for Intel PT, double-quick +decoding produces 1 sample for each PSB synchronization packet, which in turn +come after a certain number of bytes output, determined by psb_period (refer +perf Intel PT documentation). The minimum number of double-quick samples that +will define a time range can be set by the --min_size option, which defaults to +64. +""") + ap.add_argument("-o", "--output-dir", default="parallel-perf-output", + help="output directory (default 'parallel-perf-output')") + ap.add_argument("-j", "--jobs", type=int, default=0, + help="maximum number of jobs to run in parallel at one time " + "(default is the number of CPUs)") + ap.add_argument("-n", "--nr", type=int, default=0, + help="number of time subdivisions (default is the number of jobs)") + ap.add_argument("-i", "--interval", type=float, default=0, + help="subdivide the time range using this time interval " + "(in seconds e.g. 0.1 for a tenth of a second)") + ap.add_argument("-c", "--per-cpu", action="store_true", + help="process data for each CPU in parallel") + ap.add_argument("-m", "--min-interval", type=float, default=glb_min_interval, + help=f"minimum interval (default {glb_min_interval} seconds)") + ap.add_argument("-p", "--pipe-to", + help="command to pipe output to (optional)") + ap.add_argument("-N", "--no-per-cpu", action="store_true", + help="do not process data for each CPU in parallel") + ap.add_argument("-b", "--min_size", type=int, default=glb_min_samples, + help="minimum data size (for Intel PT in PSBs)") + ap.add_argument("-D", "--dry-run", action="store_true", + help="do not run any jobs, just show the perf script commands") + ap.add_argument("-q", "--quiet", action="store_true", + help="do not print any messages except errors") + ap.add_argument("-v", "--verbose", action="store_true", + help="print more messages") + ap.add_argument("-d", "--debug", action="store_true", + help="print debugging messages") + cmd_line = list(args) + try: + split_pos = cmd_line.index("--") + cmd = cmd_line[split_pos + 1:] + args = cmd_line[:split_pos] + except (OSError, ValueError, RuntimeError): + cmd = None + args = cmd_line + a = ap.parse_args(args=args[1:]) + a.cmd = cmd + setattr(a, "_verbosity", Verbosity(a.quiet, a.verbose, a.debug)) + try: + if not a.cmd: + if a.cmd is None and len(args) <= 1: + ap.print_help() + return True + raise RuntimeError( + "command line must contain '--' before perf command") + return run_parallel_perf(a) + except (OSError, ValueError, RuntimeError, KeyboardInterrupt) as e: + print("Fatal error: ", str(e)) + if a.debug: + raise + return False + + +if __name__ == "__main__": + if not main(sys.argv): + sys.exit(1) diff --git a/tools/perf/python/perf.pyi b/tools/perf/python/perf.pyi index 58b0b3ed819d..087da00728fd 100644 --- a/tools/perf/python/perf.pyi +++ b/tools/perf/python/perf.pyi @@ -20,7 +20,7 @@ def metrics() -> List[Dict[str, Union[str, List[str]]]]: """ ... -def syscall_name(id: int, *, elf_machine: Optional[int] = None) -> str: +def syscall_name(id: int, elf_machine: Optional[int] = None) -> Optional[str]: """Convert a syscall number to its name. Args: @@ -32,7 +32,7 @@ def syscall_name(id: int, *, elf_machine: Optional[int] = None) -> str: """ ... -def syscall_id(name: str, *, elf_machine: Optional[int] = None) -> int: +def syscall_id(name: str, elf_machine: Optional[int] = None) -> int: """Convert a syscall name to its number. Args: @@ -44,6 +44,18 @@ def syscall_id(name: str, *, elf_machine: Optional[int] = None) -> int: """ ... +def arch_strerrno(err: int, elf_machine: Optional[int] = None) -> Optional[str]: + """Convert an errno number to its symbolic name. + + Args: + err: The errno number (positive or negative). + elf_machine: Optional ELF machine type. + + Returns: + The symbolic name of the errno, or None if unknown. + """ + ... + def parse_events( event_string: str, cpus: Optional[cpu_map] = None, @@ -84,7 +96,7 @@ class data: class thread: """Represents a thread in the system.""" - def comm(self) -> str: + def comm(self) -> Optional[str]: """Get the command name of the thread.""" ... pid: int @@ -153,7 +165,13 @@ class evsel: def __str__(self) -> str: """Return string representation of the event.""" ... - def open(self) -> None: + def open( + self, + cpus: Optional['cpu_map'] = None, + threads: Optional[thread_map] = None, + group: bool = False, + inherit: bool = False, + ) -> None: """Open the event selector file descriptor table.""" ... def read(self, cpu: int, thread: int) -> counts_values: @@ -182,6 +200,8 @@ class _sample_members: sample_time: int sample_id: int sample_stream_id: int + machine_pid: int + vcpu: int sample_period: int sample_cpu: int @@ -197,19 +217,27 @@ class sample_event(_sample_members): sample_cyc_count: int type: int raw_buf: bytes - dso: str - dso_long_name: str - dso_bid: Optional[bytes] - map_start: int - map_end: int - map_pgoff: int - symbol: str - sym_start: int - sym_end: int + dso: Optional[str] + dso_long_name: Optional[str] + dso_bid: Optional[str] + map_start: Optional[int] + map_end: Optional[int] + map_pgoff: Optional[int] + symbol: Optional[str] + sym_start: Optional[int] + sym_end: Optional[int] + sym_offset: Optional[int] + addr_dso: Optional[str] + addr_symbol: Optional[str] + addr_sym_offset: Optional[int] + branch_type: int + in_tx: int + flags: int + transaction: int brstack: Optional['branch_stack'] callchain: Optional['callchain'] - def srccode(self) -> str: ... - def insn(self) -> str: ... + def srccode(self, addr: int = ..., /) -> Optional[tuple[Optional[str], int, Optional[str]]]: ... + def insn(self) -> Optional[bytes]: ... def __getattr__(self, name: str) -> Any: ... class mmap_event(_sample_members): @@ -286,8 +314,9 @@ class read_event(_sample_members): class switch_event(_sample_members): """Represents a SWITCH or SWITCH_CPU_WIDE record.""" type: int - next_prev_pid: int - next_prev_tid: int + misc: int + next_prev_pid: Optional[int] + next_prev_tid: Optional[int] evsel: Optional['evsel'] class branch_entry: @@ -316,17 +345,20 @@ class branch_stack: """Sequence of branch entries in the branch stack.""" def __len__(self) -> int: ... def __getitem__(self, index: int) -> branch_entry: ... + def __iter__(self) -> Iterator[branch_entry]: ... class callchain_node: """Represents a frame in the callchain.""" ip: int - symbol: Optional[str] - dso: Optional[str] + symbol: str + dso: str + sym_offset: Optional[int] class callchain: """Sequence of callchain frames.""" def __len__(self) -> int: ... def __getitem__(self, index: int) -> callchain_node: ... + def __iter__(self) -> Iterator[callchain_node]: ... class stat_event(_sample_members): """Represents a stat event from perf.""" @@ -362,10 +394,10 @@ class evlist: def close(self) -> None: """Close the events in the list.""" ... - def mmap(self) -> None: + def mmap(self, pages: int = 128, overwrite: bool = False) -> None: """Memory map the event buffers.""" ... - def poll(self, timeout: int) -> int: + def poll(self, timeout: int = -1) -> int: """Poll for events. Args: @@ -375,7 +407,7 @@ class evlist: Number of events ready. """ ... - def read_on_cpu(self, cpu: int) -> Optional[Any]: + def read_on_cpu(self, cpu: int, sample_id_all: bool = True) -> Optional[Any]: """Read a sample event from a specific CPU. Args: @@ -412,23 +444,50 @@ class evlist: def enable(self) -> None: """Enable all events in the list.""" ... - def get_pollfd(self) -> List[int]: + def get_pollfd(self) -> List[Any]: """Get a list of file descriptors for polling.""" ... - def add(self, evsel: evsel) -> int: + def add(self, evsel: evsel, /) -> int: """Add an event to the list.""" ... + def __len__(self) -> int: ... + def __getitem__(self, index: int) -> evsel: ... def __iter__(self) -> Iterator[evsel]: """Iterate over the events (evsel) in the list.""" ... +class call_return: + """Represents a call/return event from instruction trace decoding.""" + db_id: int + parent_id: int + insn_count: int + cyc_count: int + comm: Optional[str] + machine_pid: int + pid: Optional[int] + tid: Optional[int] + call_time: int + return_time: int + branch_count: int + call_ref: int + return_ref: int + flags: int + call_path: Optional[callchain] + + class session: def __init__( self, data: data, sample: Optional[Callable[[sample_event], None]] = None, - stat: Optional[Callable[[Any, Optional[str]], None]] = None + stat: Optional[Callable[[Any, Optional[str]], None]] = None, + context_switch: Optional[Callable[[switch_event], None]] = None, + call_return: Optional[Callable[[call_return], None]] = None, + itrace: Optional[str] = None, + vmlinux: Optional[str] = None, + kallsyms: Optional[str] = None, + symfs: Optional[str] = None ) -> None: """Initialize a perf session. @@ -438,10 +497,12 @@ class session: stat: Callback for stat events. """ ... + e_machine: int + is_64_bit: bool def process_events(self) -> None: """Process all events in the session.""" ... - def find_thread(self, pid: int) -> thread: + def find_thread(self, pid: int, tid: int = -1, /) -> Optional[thread]: """Returns the thread associated with a pid.""" ... @@ -670,3 +731,12 @@ RECORD_STAT_ROUND: int RECORD_MISC_SWITCH_OUT: int """MISC_SWITCH_OUT record.""" + +CALL_RETURN_NO_CALL: int +"""'return' but no matching 'call'.""" + +CALL_RETURN_NO_RETURN: int +"""'call' but no matching 'return'.""" + +CALL_RETURN_NON_CALL: int +"""A branch but not a 'call' to the start of a different symbol.""" diff --git a/tools/perf/python/perf_live.py b/tools/perf/python/perf_live.py index 616b5f657463..99e73d58a96f 100755 --- a/tools/perf/python/perf_live.py +++ b/tools/perf/python/perf_live.py @@ -33,10 +33,16 @@ class LiveSession: # Poll for events with 100ms timeout try: self.evlist.poll(100) + # CPython's PyErr_SetFromErrno() invokes PyErr_CheckSignals() on + # EINTR, converting SIGINT into KeyboardInterrupt, which inherits + # from BaseException rather than InterruptedError/OSError, so + # Ctrl-C exits the loop and triggers finally: self.evlist.close() + # while non-fatal signals retry poll(). except InterruptedError: continue for cpu in self.cpus: - for _ in range(1000): # Limit to 1000 events per CPU per poll to prevent starvation + # Limit to 1000 events per CPU per poll to prevent starvation + for _ in range(1000): try: event = self.evlist.read_on_cpu(cpu) except TypeError as e: @@ -53,7 +59,5 @@ class LiveSession: if event.type == perf.RECORD_SAMPLE: self.sample_callback(event) - except KeyboardInterrupt: - pass finally: self.evlist.close() diff --git a/tools/perf/python/powerpc-hcalls.py b/tools/perf/python/powerpc-hcalls.py new file mode 100755 index 000000000000..ab5867c05d3e --- /dev/null +++ b/tools/perf/python/powerpc-hcalls.py @@ -0,0 +1,334 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0+ +""" +Hypervisor call statistics + +Copyright (C) 2018 Ravi Bangoria, IBM Corporation +Ported from tools/perf/scripts/python/powerpc-hcalls.py +""" +from __future__ import annotations + +import argparse +from collections import defaultdict +import sys +import perf + +# Hypervisor call table +HCALL_TABLE = { + 4: 'H_REMOVE', + 8: 'H_ENTER', + 12: 'H_READ', + 16: 'H_CLEAR_MOD', + 20: 'H_CLEAR_REF', + 24: 'H_PROTECT', + 28: 'H_GET_TCE', + 32: 'H_PUT_TCE', + 36: 'H_SET_SPRG0', + 40: 'H_SET_DABR', + 44: 'H_PAGE_INIT', + 48: 'H_SET_ASR', + 52: 'H_ASR_ON', + 56: 'H_ASR_OFF', + 60: 'H_LOGICAL_CI_LOAD', + 64: 'H_LOGICAL_CI_STORE', + 68: 'H_LOGICAL_CACHE_LOAD', + 72: 'H_LOGICAL_CACHE_STORE', + 76: 'H_LOGICAL_ICBI', + 80: 'H_LOGICAL_DCBF', + 84: 'H_GET_TERM_CHAR', + 88: 'H_PUT_TERM_CHAR', + 92: 'H_REAL_TO_LOGICAL', + 96: 'H_HYPERVISOR_DATA', + 100: 'H_EOI', + 104: 'H_CPPR', + 108: 'H_IPI', + 112: 'H_IPOLL', + 116: 'H_XIRR', + 120: 'H_MIGRATE_DMA', + 124: 'H_PERFMON', + 220: 'H_REGISTER_VPA', + 224: 'H_CEDE', + 228: 'H_CONFER', + 232: 'H_PROD', + 236: 'H_GET_PPP', + 240: 'H_SET_PPP', + 244: 'H_PURR', + 248: 'H_PIC', + 252: 'H_REG_CRQ', + 256: 'H_FREE_CRQ', + 260: 'H_VIO_SIGNAL', + 264: 'H_SEND_CRQ', + 272: 'H_COPY_RDMA', + 276: 'H_REGISTER_LOGICAL_LAN', + 280: 'H_FREE_LOGICAL_LAN', + 284: 'H_ADD_LOGICAL_LAN_BUFFER', + 288: 'H_SEND_LOGICAL_LAN', + 292: 'H_BULK_REMOVE', + 304: 'H_MULTICAST_CTRL', + 308: 'H_SET_XDABR', + 312: 'H_STUFF_TCE', + 316: 'H_PUT_TCE_INDIRECT', + 332: 'H_CHANGE_LOGICAL_LAN_MAC', + 336: 'H_VTERM_PARTNER_INFO', + 340: 'H_REGISTER_VTERM', + 344: 'H_FREE_VTERM', + 348: 'H_RESET_EVENTS', + 352: 'H_ALLOC_RESOURCE', + 356: 'H_FREE_RESOURCE', + 360: 'H_MODIFY_QP', + 364: 'H_QUERY_QP', + 368: 'H_REREGISTER_PMR', + 372: 'H_REGISTER_SMR', + 376: 'H_QUERY_MR', + 380: 'H_QUERY_MW', + 384: 'H_QUERY_HCA', + 388: 'H_QUERY_PORT', + 392: 'H_MODIFY_PORT', + 396: 'H_DEFINE_AQP1', + 400: 'H_GET_TRACE_BUFFER', + 404: 'H_DEFINE_AQP0', + 408: 'H_RESIZE_MR', + 412: 'H_ATTACH_MCQP', + 416: 'H_DETACH_MCQP', + 420: 'H_CREATE_RPT', + 424: 'H_REMOVE_RPT', + 428: 'H_REGISTER_RPAGES', + 432: 'H_DISABLE_AND_GET', + 436: 'H_ERROR_DATA', + 440: 'H_GET_HCA_INFO', + 444: 'H_GET_PERF_COUNT', + 448: 'H_MANAGE_TRACE', + 456: 'H_GET_CPU_CHARACTERISTICS', + 468: 'H_FREE_LOGICAL_LAN_BUFFER', + 472: 'H_POLL_PENDING', + 484: 'H_QUERY_INT_STATE', + 580: 'H_ILLAN_ATTRIBUTES', + 584: 'H_ADD_LOGICAL_LAN_BUFFERS', + 592: 'H_MODIFY_HEA_QP', + 596: 'H_QUERY_HEA_QP', + 600: 'H_QUERY_HEA', + 604: 'H_QUERY_HEA_PORT', + 608: 'H_MODIFY_HEA_PORT', + 612: 'H_REG_BCMC', + 616: 'H_DEREG_BCMC', + 620: 'H_REGISTER_HEA_RPAGES', + 624: 'H_DISABLE_AND_GET_HEA', + 628: 'H_GET_HEA_INFO', + 632: 'H_ALLOC_HEA_RESOURCE', + 644: 'H_ADD_CONN', + 648: 'H_DEL_CONN', + 664: 'H_JOIN', + 672: 'H_VASI_SIGNAL', + 676: 'H_VASI_STATE', + 680: 'H_VIOCTL', + 688: 'H_ENABLE_CRQ', + 696: 'H_GET_EM_PARMS', + 720: 'H_SET_MPP', + 724: 'H_GET_MPP', + 732: 'H_REG_SUB_CRQ', + 736: 'H_FREE_SUB_CRQ', + 740: 'H_SEND_SUB_CRQ', + 744: 'H_SEND_SUB_CRQ_INDIRECT', + 748: 'H_HOME_NODE_ASSOCIATIVITY', + 756: 'H_BEST_ENERGY', + 764: 'H_XIRR_X', + 768: 'H_RANDOM', + 772: 'H_COP', + 788: 'H_GET_MPP_X', + 796: 'H_SET_MODE', + 808: 'H_BLOCK_REMOVE', + 856: 'H_CLEAR_HPT', + 864: 'H_REQUEST_VMC', + 876: 'H_RESIZE_HPT_PREPARE', + 880: 'H_RESIZE_HPT_COMMIT', + 892: 'H_REGISTER_PROC_TBL', + 896: 'H_SIGNAL_SYS_RESET', + 904: 'H_ALLOCATE_VAS_WINDOW', + 908: 'H_MODIFY_VAS_WINDOW', + 912: 'H_DEALLOCATE_VAS_WINDOW', + 916: 'H_QUERY_VAS_WINDOW', + 920: 'H_QUERY_VAS_CAPABILITIES', + 924: 'H_QUERY_NX_CAPABILITIES', + 928: 'H_GET_NX_FAULT', + 936: 'H_INT_GET_SOURCE_INFO', + 940: 'H_INT_SET_SOURCE_CONFIG', + 944: 'H_INT_GET_SOURCE_CONFIG', + 948: 'H_INT_GET_QUEUE_INFO', + 952: 'H_INT_SET_QUEUE_CONFIG', + 956: 'H_INT_GET_QUEUE_CONFIG', + 960: 'H_INT_SET_OS_REPORTING_LINE', + 964: 'H_INT_GET_OS_REPORTING_LINE', + 968: 'H_INT_ESB', + 972: 'H_INT_SYNC', + 976: 'H_INT_RESET', + 996: 'H_SCM_READ_METADATA', + 1000: 'H_SCM_WRITE_METADATA', + 1004: 'H_SCM_BIND_MEM', + 1008: 'H_SCM_UNBIND_MEM', + 1012: 'H_SCM_QUERY_BLOCK_MEM_BINDING', + 1016: 'H_SCM_QUERY_LOGICAL_MEM_BINDING', + 1020: 'H_SCM_UNBIND_ALL', + 1024: 'H_SCM_HEALTH', + 1048: 'H_SCM_PERFORMANCE_STATS', + 1052: 'H_PKS_GET_CONFIG', + 1056: 'H_PKS_SET_PASSWORD', + 1060: 'H_PKS_GEN_PASSWORD', + 1068: 'H_PKS_WRITE_OBJECT', + 1072: 'H_PKS_GEN_KEY', + 1076: 'H_PKS_READ_OBJECT', + 1080: 'H_PKS_REMOVE_OBJECT', + 1084: 'H_PKS_CONFIRM_OBJECT_FLUSHED', + 1096: 'H_RPT_INVALIDATE', + 1100: 'H_SCM_FLUSH', + 1104: 'H_GET_ENERGY_SCALE_INFO', + 1108: 'H_PKS_SIGNED_UPDATE', + 1112: 'H_HTM', + 1116: 'H_WATCHDOG', + # Platform specific hcalls used by KVM on PowerVM + 1120: 'H_GUEST_GET_CAPABILITIES', + 1124: 'H_GUEST_SET_CAPABILITIES', + 1136: 'H_GUEST_CREATE', + 1140: 'H_GUEST_CREATE_VCPU', + 1144: 'H_GUEST_GET_STATE', + 1148: 'H_GUEST_SET_STATE', + 1152: 'H_GUEST_RUN_VCPU', + 1156: 'H_GUEST_COPY_MEMORY', + 1160: 'H_GUEST_DELETE', + # Key wrapping hcalls + 1168: 'H_PKS_WRAP_OBJECT', + 1172: 'H_PKS_UNWRAP_OBJECT', + # Platform-specific hcalls used by the Ultravisor + 61184: 'H_SVM_PAGE_IN', + 61188: 'H_SVM_PAGE_OUT', + 61192: 'H_SVM_INIT_START', + 61196: 'H_SVM_INIT_DONE', + 61204: 'H_SVM_INIT_ABORT', + # Platform specific hcalls used by KVM + 61440: 'H_RTAS', + # Platform specific hcalls used by QEMU/SLOF + 61441: 'H_LOGICAL_MEMOP', + 61442: 'H_CAS', + 61443: 'H_UPDATE_DT', + # Platform specific hcalls provided by PHYP + 61560: 'H_GET_24X7_CATALOG_PAGE', + 61564: 'H_GET_24X7_DATA', + 61568: 'H_GET_PERF_COUNTER_INFO', + # Platform-specific hcalls used for nested HV KVM + 63488: 'H_SET_PARTITION_TABLE', + 63492: 'H_ENTER_NESTED', + 63496: 'H_TLB_INVALIDATE', + 63500: 'H_COPY_TOFROM_GUEST', +} + + +class HCallAnalyzer: + """Analyzes hypervisor calls and aggregates statistics.""" + + def __init__(self, cmd_args) -> None: + self.args = cmd_args + # output: {opcode: {'min': min, 'max': max, 'time': time, 'cnt': cnt}} + self.output: dict[int, dict[str, 'int | float']] = \ + defaultdict(lambda: {'time': 0, 'cnt': 0, 'min': float('inf'), 'max': 0}) + # d_enter: {cpu: {opcode: nsec}} + self.d_enter: dict[int, dict[int, int]] = {} + self.print_ptrn = '%-28s%10s%10s%10s%10s' + + def hcall_table_lookup(self, opcode: int) -> str: + """Lookup hcall name by opcode.""" + return HCALL_TABLE.get(opcode, str(opcode)) + + def process_event(self, sample: perf.sample_event) -> None: + """Process a single sample event.""" + name = str(sample.evsel) + if name not in ("evsel(powerpc:hcall_entry)", "evsel(powerpc:hcall_exit)"): + return + + try: + cpu = sample.sample_cpu + opcode = sample.opcode + except AttributeError: + print("ERROR: tracepoint fields missed (is libtraceevent enabled?)", + file=sys.stderr) + sys.exit(1) + + # sample.sample_time represents the nsecs + time = sample.sample_time + + if opcode < 0 or cpu < 0: + return + + if name == "evsel(powerpc:hcall_entry)": + if cpu not in self.d_enter: + self.d_enter[cpu] = {} + self.d_enter[cpu][opcode] = time + elif name == "evsel(powerpc:hcall_exit)": + if cpu in self.d_enter and opcode in self.d_enter[cpu]: + time_entry = self.d_enter[cpu][opcode] + diff = time - time_entry + del self.d_enter[cpu][opcode] + + stats = self.output[opcode] + stats['time'] += diff + stats['cnt'] += 1 + if diff < stats['min']: + stats['min'] = diff + if diff > stats['max']: + stats['max'] = diff + + def print_summary(self) -> None: + """Print aggregated statistics.""" + print(self.print_ptrn % ('hcall', 'count', 'min(ns)', 'max(ns)', 'avg(ns)')) + print('-' * 68) + + def sort_output(opcode): + stats = self.output[opcode] + sort_by = getattr(self.args, 'sort', None) + if sort_by == 'min': + return stats['min'] + if sort_by == 'max': + return stats['max'] + if sort_by == 'avg': + return stats['time'] // stats['cnt'] + return stats['cnt'] + + for opcode in sorted(self.output.keys(), key=sort_output, reverse=True): + h_name = self.hcall_table_lookup(opcode) + stats = self.output[opcode] + time = stats['time'] + cnt = stats['cnt'] + min_t = stats['min'] + max_t = stats['max'] + + # Avoid float representation for large integers if possible, + # or use formatted strings. Legacy used time//cnt. + avg_t = time // cnt if cnt > 0 else 0 + + # If min was not updated, it remains inf, but cnt should be > 0 if in output + if min_t == float('inf'): + min_t = 0 + + print(self.print_ptrn % (h_name, cnt, int(min_t), int(max_t), avg_t)) + + +if __name__ == "__main__": + ap = argparse.ArgumentParser(description="Hypervisor call statistics") + ap.add_argument("-i", "--input", default="perf.data", help="Input file name") + ap.add_argument("-s", "--sort", default="count", + choices=["count", "min", "max", "avg"], help="Sort key") + args = ap.parse_args() + + analyzer = HCallAnalyzer(args) + + session = None + try: + session = perf.session(perf.data(args.input), sample=analyzer.process_event) + session.process_events() + analyzer.print_summary() + except KeyboardInterrupt: + analyzer.print_summary() + except (OSError, ValueError, KeyError, RuntimeError, TypeError, AttributeError) as e: + print(f"Error processing events: {e}") + sys.exit(1) + finally: + # Break the reference cycle between perf.session and analyzer.process_event. + session = None diff --git a/tools/perf/python/rw-by-file.py b/tools/perf/python/rw-by-file.py new file mode 100755 index 000000000000..b33330261606 --- /dev/null +++ b/tools/perf/python/rw-by-file.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Display r/w activity for files read/written to for a given program.""" +from __future__ import annotations + +import argparse +from collections import defaultdict +import sys +from typing import Optional, Dict +import perf + +class RwByFile: + """Tracks and displays read/write activity by file descriptor.""" + def __init__(self, comm: str) -> None: + self.for_comm = comm + self.reads: Dict[int, Dict[str, int]] = defaultdict( + lambda: {"bytes_requested": 0, "total_reads": 0} + ) + self.writes: Dict[int, Dict[str, int]] = defaultdict( + lambda: {"bytes_written": 0, "total_writes": 0} + ) + self.unhandled: Dict[str, int] = defaultdict(int) + self.session: Optional[perf.session] = None + + def process_event(self, sample: perf.sample_event) -> None: + """Process events.""" + event_name = str(sample.evsel) + if event_name.startswith("evsel(") and event_name.endswith(")"): + event_name = event_name[6:-1] + event_name = "".join(c if c.isprintable() else "?" for c in event_name) + + pid = sample.sample_pid + assert self.session is not None + try: + thread = self.session.find_thread(pid, sample.sample_tid) + comm = (thread.comm() if thread else None) or "unknown" + except (TypeError, AttributeError): + comm = "unknown" + + if comm != self.for_comm: + return + + if event_name == "syscalls:sys_enter_read": + try: + fd = sample.fd + count = sample.count + self.reads[fd]["bytes_requested"] += count + self.reads[fd]["total_reads"] += 1 + except AttributeError: + self.unhandled[event_name] += 1 + elif event_name == "syscalls:sys_enter_write": + try: + fd = sample.fd + count = sample.count + self.writes[fd]["bytes_written"] += count + self.writes[fd]["total_writes"] += 1 + except AttributeError: + self.unhandled[event_name] += 1 + else: + self.unhandled[event_name] += 1 + + def print_totals(self) -> None: + """Print summary tables.""" + print(f"file read counts for {self.for_comm}:\n") + print(f"{'fd':>6s} {'# reads':>10s} {'bytes_requested':>15s}") + print(f"{'-'*6} {'-'*10} {'-'*15}") + + for fd, data in sorted(self.reads.items(), + key=lambda kv: kv[1]["bytes_requested"], reverse=True): + print(f"{fd:6d} {data['total_reads']:10d} {data['bytes_requested']:15d}") + + print(f"\nfile write counts for {self.for_comm}:\n") + print(f"{'fd':>6s} {'# writes':>10s} {'bytes_written':>15s}") + print(f"{'-'*6} {'-'*10} {'-'*15}") + + for fd, data in sorted(self.writes.items(), + key=lambda kv: kv[1]["bytes_written"], reverse=True): + print(f"{fd:6d} {data['total_writes']:10d} {data['bytes_written']:15d}") + + if self.unhandled: + print("\nunhandled events:\n") + print(f"{'event':<40s} {'count':>10s}") + print(f"{'-'*40} {'-'*10}") + for event_name, count in self.unhandled.items(): + print(f"{event_name:<40s} {count:10d}") + + def run(self, input_file: str) -> None: + """Run the session.""" + self.session = perf.session(perf.data(input_file), sample=self.process_event) + try: + self.session.process_events() + finally: + # Break the reference cycle between perf.session and self.process_event + # because perf.session lacks cyclic GC support (tp_traverse). + self.session = None + self.print_totals() + +def main() -> None: + """Main function.""" + parser = argparse.ArgumentParser(description="Trace r/w activity by file") + parser.add_argument("comm", help="Filter by command name") + parser.add_argument("-i", "--input", default="perf.data", help="Input file") + args = parser.parse_args() + + analyzer = RwByFile(args.comm) + try: + analyzer.run(args.input) + except IOError as e: + print(e, file=sys.stderr) + sys.exit(1) + +if __name__ == "__main__": + main() diff --git a/tools/perf/python/rw-by-pid.py b/tools/perf/python/rw-by-pid.py new file mode 100755 index 000000000000..8e2d1dbd9a08 --- /dev/null +++ b/tools/perf/python/rw-by-pid.py @@ -0,0 +1,205 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Display r/w activity for all processes.""" +from __future__ import annotations + +import argparse +from collections import defaultdict +import sys +from typing import Optional, Dict, List, Tuple, Any +import perf + +class RwByPid: + """Tracks and displays read/write activity by PID.""" + def __init__(self) -> None: + self.reads: Dict[int, Dict[str, Any]] = defaultdict( + lambda: { + "bytes_requested": 0, + "bytes_read": 0, + "total_reads": 0, + "comm": "", + "errors": defaultdict(int), + } + ) + self.writes: Dict[int, Dict[str, Any]] = defaultdict( + lambda: { + "bytes_requested": 0, + "bytes_written": 0, + "total_writes": 0, + "comm": "", + "errors": defaultdict(int), + } + ) + self.unhandled: Dict[str, int] = defaultdict(int) + self.session: Optional[perf.session] = None + + def _handle_sys_enter_read(self, pid: int, comm: str, sample: perf.sample_event) -> None: + try: + count = sample.count + self.reads[pid]["bytes_requested"] += count + self.reads[pid]["total_reads"] += 1 + self.reads[pid]["comm"] = comm + except AttributeError: + self.unhandled["sys_enter_read_attr_err"] += 1 + + def _handle_sys_exit_read(self, pid: int, sample: perf.sample_event, comm: str = "") -> None: + try: + ret = sample.ret + # Convert unsigned 64-bit or 32-bit tracepoint return values to signed integers. + # Because kernel read/write transfers are bounded by MAX_RW_COUNT (0x7ffff000 < 2 GiB), + # 0xfffff000..0xffffffff is always a 32-bit negative errno even on 64-bit hosts. + if ret >= 0x8000000000000000: + ret -= 0x10000000000000000 + elif 0xfffff000 <= ret <= 0xffffffff: + ret -= 0x100000000 + if not self.reads[pid]["comm"] and comm: + self.reads[pid]["comm"] = comm + if ret >= 0: + self.reads[pid]["bytes_read"] += ret + else: + self.reads[pid]["errors"][ret] += 1 + except AttributeError: + self.unhandled["sys_exit_read_attr_err"] += 1 + + def _handle_sys_enter_write(self, pid: int, comm: str, sample: perf.sample_event) -> None: + try: + count = sample.count + self.writes[pid]["bytes_requested"] += count + self.writes[pid]["total_writes"] += 1 + self.writes[pid]["comm"] = comm + except AttributeError: + self.unhandled["sys_enter_write_attr_err"] += 1 + + def _handle_sys_exit_write(self, pid: int, sample: perf.sample_event, comm: str = "") -> None: + try: + ret = sample.ret + # Convert unsigned 64-bit or 32-bit tracepoint return values to signed integers. + # Because kernel read/write transfers are bounded by MAX_RW_COUNT (0x7ffff000 < 2 GiB), + # 0xfffff000..0xffffffff is always a 32-bit negative errno even on 64-bit hosts. + if ret >= 0x8000000000000000: + ret -= 0x10000000000000000 + elif 0xfffff000 <= ret <= 0xffffffff: + ret -= 0x100000000 + if not self.writes[pid]["comm"] and comm: + self.writes[pid]["comm"] = comm + if ret >= 0: + self.writes[pid]["bytes_written"] += ret + else: + self.writes[pid]["errors"][ret] += 1 + except AttributeError: + self.unhandled["sys_exit_write_attr_err"] += 1 + + def process_event(self, sample: perf.sample_event) -> None: + """Process events.""" + event_name = str(sample.evsel) + if event_name.startswith("evsel(") and event_name.endswith(")"): + event_name = event_name[6:-1] + event_name = "".join(c if c.isprintable() else "?" for c in event_name) + pid = sample.sample_pid + + assert self.session is not None + try: + thread = self.session.find_thread(pid, sample.sample_tid) + comm = (thread.comm() if thread else None) or "unknown" + except (TypeError, AttributeError): + comm = "unknown" + comm = "".join(c if c.isprintable() else "?" for c in comm) + + if event_name == "syscalls:sys_enter_read": + self._handle_sys_enter_read(pid, comm, sample) + elif event_name == "syscalls:sys_exit_read": + self._handle_sys_exit_read(pid, sample, comm) + elif event_name == "syscalls:sys_enter_write": + self._handle_sys_enter_write(pid, comm, sample) + elif event_name == "syscalls:sys_exit_write": + self._handle_sys_exit_write(pid, sample, comm) + else: + self.unhandled[event_name] += 1 + + def print_totals(self) -> None: + """Print summary tables.""" + print("read counts by pid:\n") + print( + f"{'pid':>6s} {'comm':<20s} {'# reads':>10s} " + f"{'bytes_requested':>15s} {'bytes_read':>10s}" + ) + print(f"{'-'*6} {'-'*20} {'-'*10} {'-'*15} {'-'*10}") + + for pid, data in sorted(self.reads.items(), + key=lambda kv: kv[1]["bytes_read"], reverse=True): + print( + f"{pid:6d} {data['comm']:<20s} {data['total_reads']:10d} " + f"{data['bytes_requested']:15d} {data['bytes_read']:10d}" + ) + + print("\nfailed reads by pid:\n") + print(f"{'pid':>6s} {'comm':<20s} {'error #':>7s} {'# errors':>10s}") + print(f"{'-'*6} {'-'*20} {'-'*7} {'-'*10}") + + errcounts: List[Tuple[int, str, int, int]] = [] + for pid, data in self.reads.items(): + for error, count in data["errors"].items(): + errcounts.append((pid, data["comm"], error, count)) + + for pid, comm, error, count in sorted(errcounts, key=lambda x: x[3], reverse=True): + print(f"{pid:6d} {comm:<20s} {error:7d} {count:10d}") + + print("\nwrite counts by pid:\n") + print( + f"{'pid':>6s} {'comm':<20s} {'# writes':>10s} " + f"{'bytes_requested':>15s} {'bytes_written':>15s}" + ) + print(f"{'-'*6} {'-'*20} {'-'*10} {'-'*15} {'-'*15}") + + for pid, data in sorted(self.writes.items(), + key=lambda kv: kv[1]["bytes_written"], reverse=True): + print( + f"{pid:6d} {data['comm']:<20s} {data['total_writes']:10d} " + f"{data['bytes_requested']:15d} {data['bytes_written']:15d}" + ) + + print("\nfailed writes by pid:\n") + print(f"{'pid':>6s} {'comm':<20s} {'error #':>7s} {'# errors':>10s}") + print(f"{'-'*6} {'-'*20} {'-'*7} {'-'*10}") + + errcounts = [] + for pid, data in self.writes.items(): + for error, count in data["errors"].items(): + errcounts.append((pid, data["comm"], error, count)) + + for pid, comm, error, count in sorted(errcounts, key=lambda x: x[3], reverse=True): + print(f"{pid:6d} {comm:<20s} {error:7d} {count:10d}") + + if self.unhandled: + print("\nunhandled events:\n") + print(f"{'event':<40s} {'count':>10s}") + print(f"{'-'*40} {'-'*10}") + for event_name, count in self.unhandled.items(): + print(f"{event_name:<40s} {count:10d}") + + def run(self, input_file: str) -> None: + """Run the session.""" + self.session = perf.session(perf.data(input_file), sample=self.process_event) + try: + self.session.process_events() + finally: + # Break the reference cycle between perf.session and self.process_event + # because perf.session lacks cyclic GC support (tp_traverse). + self.session = None + self.print_totals() + +def main() -> None: + """Main function.""" + parser = argparse.ArgumentParser(description="Trace r/w activity by PID") + parser.add_argument("-i", "--input", default="perf.data", help="Input file") + args = parser.parse_args() + + analyzer = RwByPid() + try: + analyzer.run(args.input) + except IOError as e: + print(e, file=sys.stderr) + sys.exit(1) + +if __name__ == "__main__": + main() diff --git a/tools/perf/python/rwtop.py b/tools/perf/python/rwtop.py new file mode 100755 index 000000000000..86674d7c8f6f --- /dev/null +++ b/tools/perf/python/rwtop.py @@ -0,0 +1,256 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Periodically displays system-wide r/w call activity, broken down by pid.""" +from __future__ import annotations + +import argparse +from collections import defaultdict +import os +import sys +from typing import Optional, Dict, Any +import perf +from perf_live import LiveSession + +class RwTop: + """Periodically displays system-wide r/w call activity.""" + def __init__(self, interval: int = 3, nlines: int = 20) -> None: + self.offline = False + self.interval_ns = interval * 1000000000 + self.nlines = nlines + self.reads: Dict[int, Dict[str, Any]] = defaultdict( + lambda: { + "bytes_requested": 0, + "bytes_read": 0, + "total_reads": 0, + "comm": "", + "errors": defaultdict(int), + } + ) + self.writes: Dict[int, Dict[str, Any]] = defaultdict( + lambda: { + "bytes_requested": 0, + "bytes_written": 0, + "total_writes": 0, + "comm": "", + "errors": defaultdict(int), + } + ) + self.unhandled: Dict[str, int] = defaultdict(int) + self.comm_cache: Dict[int, str] = {} + self.session: Optional[perf.session] = None + self.last_print_time: int = 0 + + def get_comm(self, pid: int, tid: Optional[int] = None) -> str: + """Resolve and cache the comm(and) of a pid.""" + comm = None + if self.session: + # In offline mode, query session.find_thread() directly so PERF_RECORD_COMM + # updates after execve are reflected immediately instead of returning a + # stale pre-exec comm from comm_cache. + try: + thread = (self.session.find_thread(pid, tid) + if tid is not None else self.session.find_thread(pid)) + comm = thread.comm() if thread else None + except (TypeError, AttributeError): + pass + if not comm: + comm = self.comm_cache.get(pid) + else: + comm = self.comm_cache.get(pid) + if comm: + return comm + try: + with open(f"/proc/{pid}/comm", "r", encoding="utf-8", errors="replace") as f: + comm = f.read().strip() + except OSError: + # The thread may have exited before /proc could be read. + pass + if not comm: + comm = f"PID({pid})" + comm = ''.join(c if c.isprintable() else '?' for c in comm) + self.comm_cache[pid] = comm + return comm + + def process_event(self, sample: perf.sample_event) -> None: + """Process events.""" + event_name = str(sample.evsel) + if event_name.startswith("evsel(") and event_name.endswith(")"): + event_name = event_name[6:-1] + event_name = "".join(c if c.isprintable() else "?" for c in event_name) + pid = sample.sample_pid + sample_time = getattr(sample, "sample_time", 0) or 0 + + if sample_time > 0: + if self.last_print_time == 0: + self.last_print_time = sample_time + elif (sample_time > self.last_print_time and + sample_time - self.last_print_time >= self.interval_ns): + self.print_totals() + self.last_print_time = sample_time + + # Map each event onto the totals it updates. "enter" events count the + # requested bytes, "exit" events the transferred bytes or the error. + handlers = { + "syscalls:sys_enter_read": (self.reads, "total_reads", None), + "syscalls:sys_exit_read": (self.reads, None, "bytes_read"), + "syscalls:sys_enter_write": (self.writes, "total_writes", None), + "syscalls:sys_exit_write": (self.writes, None, "bytes_written"), + } + handler = handlers.get(event_name) + if not handler: + self.unhandled[event_name] += 1 + return + + totals, count_key, bytes_key = handler + try: + value = sample.count if count_key else sample.ret + except AttributeError: + self.unhandled[event_name] += 1 + return + + data = totals[pid] + data["comm"] = self.get_comm(pid, getattr(sample, "sample_tid", None)) + if count_key: + data["bytes_requested"] += value + data[count_key] += 1 + elif bytes_key: + # Convert unsigned 32-bit or 64-bit kernel error return values to signed integers. + # Because kernel read/write transfers are bounded by MAX_RW_COUNT (0x7ffff000 < 2 GiB), + # 0xfffff000..0xffffffff is always a 32-bit negative errno even on 64-bit hosts. + if value > 0: + if 0xfffff000 <= value <= 0xffffffff: + value -= 0x100000000 + elif value >= 0x8000000000000000: + value -= 0x10000000000000000 + if value >= 0: + data[bytes_key] += value + else: + data["errors"][value] += 1 + + def print_totals(self) -> None: + """Print summary tables.""" + if not self.offline: + print('\x1b[H\x1b[2J', end='') + print("read counts by pid:\n") + print( + f"{'pid':>6s} {'comm':<20s} {'# reads':>10s} " + f"{'bytes_req':>10s} {'bytes_read':>10s}" + ) + print(f"{'-'*6} {'-'*20} {'-'*10} {'-'*10} {'-'*10}") + + count = 0 + for pid, data in sorted(self.reads.items(), + key=lambda kv: kv[1]["bytes_read"], reverse=True): + print( + f"{pid:6d} {data['comm']:<20s} {data['total_reads']:10d} " + f"{data['bytes_requested']:10d} {data['bytes_read']:10d}" + ) + count += 1 + if count >= self.nlines: + break + + print("\nfailed reads by pid:\n") + print(f"{'pid':>6s} {'comm':<20s} {'error #':>7s} {'# errors':>10s}") + print(f"{'-'*6} {'-'*20} {'-'*7} {'-'*10}") + + errcounts = [] + for pid, data in self.reads.items(): + for error, cnt in data["errors"].items(): + errcounts.append((pid, data["comm"], error, cnt)) + + sorted_errcounts = sorted(errcounts, key=lambda x: x[3], reverse=True) + for pid, comm, error, cnt in sorted_errcounts[:self.nlines]: + print(f"{pid:6d} {comm:<20s} {error:7d} {cnt:10d}") + + print("\nwrite counts by pid:\n") + print( + f"{'pid':>6s} {'comm':<20s} {'# writes':>10s} " + f"{'bytes_req':>10s} {'bytes_written':>13s}" + ) + print(f"{'-'*6} {'-'*20} {'-'*10} {'-'*10} {'-'*13}") + + count = 0 + for pid, data in sorted(self.writes.items(), + key=lambda kv: kv[1]["bytes_written"], reverse=True): + print( + f"{pid:6d} {data['comm']:<20s} {data['total_writes']:10d} " + f"{data['bytes_requested']:10d} {data['bytes_written']:13d}" + ) + count += 1 + if count >= self.nlines: + break + + print("\nfailed writes by pid:\n") + print(f"{'pid':>6s} {'comm':<20s} {'error #':>7s} {'# errors':>10s}") + print(f"{'-'*6} {'-'*20} {'-'*7} {'-'*10}") + + errcounts = [] + for pid, data in self.writes.items(): + for error, cnt in data["errors"].items(): + errcounts.append((pid, data["comm"], error, cnt)) + + sorted_errcounts = sorted(errcounts, key=lambda x: x[3], reverse=True) + for pid, comm, error, cnt in sorted_errcounts[:self.nlines]: + print(f"{pid:6d} {comm:<20s} {error:7d} {cnt:10d}") + + # Reset counts + self.reads.clear() + self.writes.clear() + self.comm_cache.clear() + + def run(self, input_file: str) -> None: + """Run the session.""" + self.session = perf.session(perf.data(input_file), sample=self.process_event) + try: + self.session.process_events() + finally: + # Break the reference cycle between perf.session and self.process_event + # because perf.session lacks cyclic GC support (tp_traverse). + self.session = None + + # Print final totals if there are any left + if self.reads or self.writes: + self.print_totals() + + if self.unhandled: + print("\nunhandled events:\n") + print(f"{'event':<40s} {'count':>10s}") + print(f"{'-'*40} {'-'*10}") + for event_name, count in self.unhandled.items(): + print(f"{event_name:<40s} {count:10d}") + +def main() -> None: + """Main function.""" + parser = argparse.ArgumentParser(description="Trace r/w activity by PID") + parser.add_argument( + "interval", type=int, nargs="?", default=3, help="Refresh interval in seconds" + ) + parser.add_argument("-i", "--input", default="perf.data", help="Input file") + parser.add_argument("-l", "--live", action="store_true", help="Run in live mode") + args = parser.parse_args() + + analyzer = RwTop(args.interval) + try: + if args.live or (not os.path.exists(args.input) and args.input == "perf.data"): + # Live mode + events = ( + "syscalls:sys_enter_read,syscalls:sys_exit_read," + "syscalls:sys_enter_write,syscalls:sys_exit_write" + ) + live_session = LiveSession(events, sample_callback=analyzer.process_event) + print("Live mode started. Press Ctrl+C to stop.", file=sys.stderr) + live_session.run() + else: + analyzer.offline = True + analyzer.run(args.input) + except IOError as e: + print(e, file=sys.stderr) + sys.exit(1) + except KeyboardInterrupt: + if not analyzer.offline: + print("\nStopping live mode...", file=sys.stderr) + if analyzer.reads or analyzer.writes: + analyzer.print_totals() + +if __name__ == "__main__": + main() diff --git a/tools/perf/python/sched-migration.py b/tools/perf/python/sched-migration.py new file mode 100755 index 000000000000..d92c1a80d42f --- /dev/null +++ b/tools/perf/python/sched-migration.py @@ -0,0 +1,496 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +""" +Cpu task migration overview toy + +Copyright (C) 2010 Frederic Weisbecker <fweisbec@gmail.com> +Ported to modern directory structure and refactored to use class. +""" +from __future__ import annotations + +import argparse +from collections import defaultdict, UserList +import importlib +import sys +from typing import Any +import perf + + +# Global threads dictionary +threads = defaultdict(lambda: "unknown") +threads[0] = "idle" + + +def thread_name(pid: int) -> str: + """Return thread name formatted with pid.""" + return f"{threads[pid]}:{pid}" + + +def task_state(state: int) -> str: + """Map task state integer to string.""" + states = { + 0: "R", + 1: "S", + 2: "D", + 64: "DEAD" + } + return states.get(state, "Unknown") + + +class RunqueueEventUnknown: + """Unknown runqueue event.""" + @staticmethod + def color(): + """Return color for event.""" + return None + + def __repr__(self): + return "unknown" + + +class RunqueueEventSleep: + """Sleep runqueue event.""" + @staticmethod + def color(): + """Return color for event.""" + return 0, 0, 0xff + + def __init__(self, sleeper: int): + self.sleeper = sleeper + + def __repr__(self): + return f"{thread_name(self.sleeper)} gone to sleep" + + +class RunqueueEventWakeup: + """Wakeup runqueue event.""" + @staticmethod + def color(): + """Return color for event.""" + return 0xff, 0xff, 0 + + def __init__(self, wakee: int): + self.wakee = wakee + + def __repr__(self): + return f"{thread_name(self.wakee)} woke up" + + +class RunqueueEventFork: + """Fork runqueue event.""" + @staticmethod + def color(): + """Return color for event.""" + return 0, 0xff, 0 + + def __init__(self, child: int): + self.child = child + + def __repr__(self): + return f"new forked task {thread_name(self.child)}" + + +class RunqueueMigrateIn: + """Migrate in runqueue event.""" + @staticmethod + def color(): + """Return color for event.""" + return 0, 0xf0, 0xff + + def __init__(self, new: int): + self.new = new + + def __repr__(self): + return f"task migrated in {thread_name(self.new)}" + + +class RunqueueMigrateOut: + """Migrate out runqueue event.""" + @staticmethod + def color(): + """Return color for event.""" + return 0xff, 0, 0xff + + def __init__(self, old: int): + self.old = old + + def __repr__(self): + return f"task migrated out {thread_name(self.old)}" + + +class RunqueueSnapshot: + """Snapshot of runqueue state.""" + + def __init__(self, tasks=None, event=None): + if tasks is None: + tasks = (0,) + if event is None: + event = RunqueueEventUnknown() + self.tasks = tuple(tasks) + self.event = event + + def sched_switch(self, prev: int, prev_state: int, next_pid: int): + """Handle sched switch in snapshot.""" + if task_state(prev_state) == "R" and next_pid in self.tasks \ + and prev in self.tasks: + return self + + event = ( + RunqueueEventSleep(prev) + if task_state(prev_state) != "R" + else RunqueueEventUnknown() + ) + + next_tasks = list(self.tasks[:]) + if prev in self.tasks: + if task_state(prev_state) != "R": + next_tasks.remove(prev) + elif task_state(prev_state) == "R": + next_tasks.append(prev) + + if next_pid not in next_tasks: + next_tasks.append(next_pid) + + return RunqueueSnapshot(next_tasks, event) + + def migrate_out(self, old: int): + """Handle task migrate out in snapshot.""" + if old not in self.tasks: + return self + next_tasks = [task for task in self.tasks if task != old] + + return RunqueueSnapshot(next_tasks, RunqueueMigrateOut(old)) + + def __migrate_in(self, new: int, event): + if new in self.tasks: + return RunqueueSnapshot(self.tasks, event) + next_tasks = self.tasks + tuple([new]) + + return RunqueueSnapshot(next_tasks, event) + + def migrate_in(self, new: int): + """Handle task migrate in snapshot.""" + return self.__migrate_in(new, RunqueueMigrateIn(new)) + + def wake_up(self, new: int): + """Handle task wakeup in snapshot.""" + return self.__migrate_in(new, RunqueueEventWakeup(new)) + + def wake_up_new(self, new: int): + """Handle task fork in snapshot.""" + return self.__migrate_in(new, RunqueueEventFork(new)) + + def load(self) -> int: + """Provide the number of tasks on the runqueue. Don't count idle""" + return len(self.tasks) - 1 + + def __repr__(self): + return self.tasks.__repr__() + + +class TimeSlice: + """Represents a time slice of execution.""" + + def __init__(self, start: int, prev): + self.start = start + self.prev = prev + self.end = start + # cpus that triggered the event + self.event_cpus: list[int] = [] + if prev is not None: + self.total_load = prev.total_load + self.rqs = prev.rqs.copy() + else: + self.rqs = defaultdict(RunqueueSnapshot) + self.total_load = 0 + + def __update_total_load(self, old_rq: RunqueueSnapshot, new_rq: RunqueueSnapshot): + diff = new_rq.load() - old_rq.load() + self.total_load += diff + + def sched_switch(self, ts_list, prev: int, prev_state: int, next_pid: int, cpu: int): + """Process sched_switch in time slice.""" + old_rq = self.prev.rqs[cpu] + new_rq = old_rq.sched_switch(prev, prev_state, next_pid) + + if old_rq is new_rq: + return + + self.rqs[cpu] = new_rq + self.__update_total_load(old_rq, new_rq) + ts_list.append(self) + self.event_cpus = [cpu] + + def migrate(self, ts_list, new: int, old_cpu: int, new_cpu: int): + """Process task migration in time slice.""" + if old_cpu == new_cpu: + return + old_rq = self.prev.rqs[old_cpu] + out_rq = old_rq.migrate_out(new) + self.rqs[old_cpu] = out_rq + self.__update_total_load(old_rq, out_rq) + + new_rq = self.prev.rqs[new_cpu] + in_rq = new_rq.migrate_in(new) + self.rqs[new_cpu] = in_rq + self.__update_total_load(new_rq, in_rq) + + ts_list.append(self) + + if old_rq is not out_rq: + self.event_cpus.append(old_cpu) + self.event_cpus.append(new_cpu) + + def wake_up(self, ts_list, pid: int, cpu: int, fork: bool): + """Process wakeup in time slice.""" + old_rq = self.prev.rqs[cpu] + if fork: + new_rq = old_rq.wake_up_new(pid) + else: + new_rq = old_rq.wake_up(pid) + + if new_rq is old_rq: + return + self.rqs[cpu] = new_rq + self.__update_total_load(old_rq, new_rq) + ts_list.append(self) + self.event_cpus = [cpu] + + def next(self, t: int): + """Create next time slice.""" + self.end = t + return TimeSlice(t, self) + + +class TimeSliceList(UserList): + """List of time slices with search capabilities.""" + + def __init__(self, arg=None): + super().__init__(arg if arg is not None else []) + self.root_win = None + + def get_time_slice(self, ts: int) -> TimeSlice: + """Get or create time slice for timestamp.""" + if len(self.data) == 0: + ts_slice = TimeSlice(ts, TimeSlice(-1, None)) + else: + ts_slice = self.data[-1].next(ts) + return ts_slice + + def find_time_slice(self, ts: int) -> int: + """Binary search for time slice containing timestamp.""" + if not self.data: + return -1 + start = 0 + end = len(self.data) + found = -1 + searching = True + while searching: + if start in (end, end - 1): + searching = False + + i = (end + start) // 2 + if self.data[i].start <= ts <= self.data[i].end: + found = i + break + + if self.data[i].end < ts: + start = i + elif self.data[i].start > ts: + end = i + + return found + + def set_root_win(self, win): + """Set root window for GUI.""" + self.root_win = win + + def mouse_down(self, cpu: int, t: int): + """Handle mouse down event from GUI.""" + idx = self.find_time_slice(t) + if idx == -1: + return + + ts = self[idx] + rq = ts.rqs[cpu] + raw = f"CPU: {cpu}\n" + raw += f"Last event : {repr(rq.event)}\n" + raw += f"Timestamp : {ts.start // (10 ** 9)}.{ts.start % (10 ** 9) // 1000:06d}\n" + raw += f"Duration : {(ts.end - ts.start) // (10 ** 3):6d} us\n" + raw += f"Load = {rq.load()}\n" + for task in rq.tasks: + raw += f"{thread_name(task)} \n" + + if self.root_win: + self.root_win.update_summary(raw) + + def update_rectangle_cpu(self, slice_obj: TimeSlice, cpu: int): + """Update rectangle for CPU in GUI.""" + rq = slice_obj.rqs[cpu] + + if slice_obj.total_load != 0: + load_rate = rq.load() / float(slice_obj.total_load) + else: + load_rate = 0 + + red_power = int(0xff - (0xff * load_rate)) + color = (0xff, red_power, red_power) + + top_color = None + if cpu in slice_obj.event_cpus: + top_color = rq.event.color() + + if self.root_win: + self.root_win.paint_rectangle_zone(cpu, color, top_color, + slice_obj.start, slice_obj.end) + + def fill_zone(self, start: int, end: int): + """Fill zone in GUI.""" + i = self.find_time_slice(start) + if i == -1: + return + + for idx in range(i, len(self.data)): + timeslice = self.data[idx] + if timeslice.start > end: + return + + for cpu in timeslice.rqs: + self.update_rectangle_cpu(timeslice, cpu) + + def interval(self) -> tuple[int, int]: + """Return start and end timestamps.""" + if len(self.data) == 0: + return 0, 0 + return self.data[0].start, self.data[-1].end + + def nr_rectangles(self) -> int: + """Return maximum CPU number.""" + if not self.data: + return 0 + last_ts = self.data[-1] + max_cpu = 0 + for cpu in last_ts.rqs: + max_cpu = max(max_cpu, cpu) + return max_cpu + + +class SchedMigrationAnalyzer: + """Analyzes task migrations and manages time slices.""" + + def __init__(self): + self.current_tsk = defaultdict(lambda: -1) + self.timeslices = TimeSliceList() + + def sched_switch(self, time: int, cpu: int, prev_comm: str, prev_pid: int, prev_state: int, + next_comm: str, next_pid: int): + """Handle sched_switch event.""" + on_cpu_task = self.current_tsk[cpu] + + if on_cpu_task not in (-1, prev_pid): + print(f"Sched switch event rejected ts: {time} cpu: {cpu} " + f"prev: {prev_comm}({prev_pid}) next: {next_comm}({next_pid})") + + threads[prev_pid] = prev_comm + threads[next_pid] = next_comm + self.current_tsk[cpu] = next_pid + + ts = self.timeslices.get_time_slice(time) + ts.sched_switch(self.timeslices, prev_pid, prev_state, next_pid, cpu) + + def migrate(self, time: int, pid: int, orig_cpu: int, dest_cpu: int): + """Handle sched_migrate_task event.""" + ts = self.timeslices.get_time_slice(time) + ts.migrate(self.timeslices, pid, orig_cpu, dest_cpu) + + def wake_up(self, time: int, pid: int, success: int, target_cpu: int, fork: bool): + """Handle wakeup event.""" + if success == 0: + return + ts = self.timeslices.get_time_slice(time) + ts.wake_up(self.timeslices, pid, target_cpu, fork) + + def process_event(self, sample: perf.sample_event) -> None: + """Collect events and pass to analyzer.""" + name = str(sample.evsel) + if name.startswith("evsel(") and name.endswith(")"): + name = name[6:-1] + if name.count(":") >= 2: + name = ":".join(name.split(":")[:2]) + time = sample.sample_time + cpu = sample.sample_cpu + _pid = sample.sample_pid + _comm = "Unknown" + + if name.startswith("sched:sched_switch"): + prev_comm = getattr(sample, "prev_comm", "Unknown") + prev_pid = getattr(sample, "prev_pid", -1) + prev_state = getattr(sample, "prev_state", 0) + next_comm = getattr(sample, "next_comm", "Unknown") + next_pid = getattr(sample, "next_pid", -1) + self.sched_switch(time, cpu, prev_comm, prev_pid, prev_state, next_comm, next_pid) + elif name.startswith("sched:sched_migrate_task"): + task_pid = getattr(sample, "pid", -1) + orig_cpu = getattr(sample, "orig_cpu", -1) + dest_cpu = getattr(sample, "dest_cpu", -1) + self.migrate(time, task_pid, orig_cpu, dest_cpu) + elif name.startswith("sched:sched_wakeup_new"): + task_pid = getattr(sample, "pid", -1) + success = getattr(sample, "success", 1) + target_cpu = getattr(sample, "target_cpu", -1) + self.wake_up(time, task_pid, success, target_cpu, True) + elif name.startswith("sched:sched_wakeup"): + task_pid = getattr(sample, "pid", -1) + success = getattr(sample, "success", 1) + target_cpu = getattr(sample, "target_cpu", -1) + self.wake_up(time, task_pid, success, target_cpu, False) + + def run_gui(self): + """Start wxPython GUI.""" + try: + wx_mod: Any = importlib.import_module("wx") + sched_gui: Any = importlib.import_module("SchedGui") + except ImportError: + print("wxPython is not available. Cannot start GUI.") + return + app = wx_mod.App(False) + _frame = sched_gui.RootFrame(self.timeslices, "Migration") + app.MainLoop() + + +if __name__ == "__main__": + ap = argparse.ArgumentParser(description="Cpu task migration overview toy") + ap.add_argument("-i", "--input", default="perf.data", help="Input file name") + ap.add_argument("-v", "--verbose", action="store_true", + help="Print parsed migration summary") + ap.add_argument("--gui", action="store_true", default=True, + help="Start the GUI (default)") + ap.add_argument("--no-gui", action="store_true", help="Do not start the GUI") + args = ap.parse_args() + + analyzer = SchedMigrationAnalyzer() + session = None + + try: + try: + session = perf.session(perf.data(args.input), sample=analyzer.process_event) + try: + session.process_events() + except KeyboardInterrupt: + pass + finally: + # Release the perf.session reference before launching the long-running + # wxPython GUI event loop so the underlying C session and callback + # reference cycle are freed immediately. + session = None + if args.verbose: + start_ts, end_ts = analyzer.timeslices.interval() + print(f"Timeslices: {len(analyzer.timeslices.data)} " + f"(interval: {start_ts}..{end_ts})") + if args.gui and not args.no_gui: + analyzer.run_gui() + except KeyboardInterrupt: + pass + except OSError as e: + print(f"Error processing events: {e}") + sys.exit(1) diff --git a/tools/perf/python/sctop.py b/tools/perf/python/sctop.py new file mode 100755 index 000000000000..48320af8b755 --- /dev/null +++ b/tools/perf/python/sctop.py @@ -0,0 +1,253 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +""" +System call top + +Periodically displays system-wide system call totals, broken down by +syscall. If a [comm] arg is specified, only syscalls called by +[comm] are displayed. If an [interval] arg is specified, the display +will be refreshed every [interval] seconds. The default interval is +3 seconds. + +Ported from tools/perf/scripts/python/sctop.py +""" +from __future__ import annotations + +import argparse +from collections import defaultdict +import os +import sys +import threading +from typing import Optional +import perf +from perf_live import LiveSession + + +class SCTopAnalyzer: + """Periodically displays system-wide system call totals.""" + + def __init__(self, for_comm: Optional[str], interval: int, offline: bool = False): + self.for_comm = for_comm + self.interval = interval + self.syscalls: dict[int, int] = defaultdict(int) + self.comm_cache: dict[int, str] = {} + self.lock = threading.Lock() + self.stop_event = threading.Event() + self.thread = threading.Thread(target=self.print_syscall_totals) + self.offline = offline + self.own_pid = os.getpid() + self.last_print_time: Optional[int] = None + self.printed = False + self.session: Optional[perf.session] = None + self.e_machine: Optional[int] = None + + def syscall_name(self, syscall_id: int) -> str: + """Lookup syscall name by ID.""" + # Mask out the x86_64 x32 ABI bit (__X32_SYSCALL_BIT = 0x40000000) before + # resolving the syscall number in the architecture's syscall table. + raw_sc_id = syscall_id & ~0x40000000 + try: + e_machine = getattr(self.session, "e_machine", self.e_machine) + if e_machine is not None: + name = perf.syscall_name(raw_sc_id, e_machine) + else: + name = perf.syscall_name(raw_sc_id) + if name is not None: + return name + except (TypeError, OverflowError): + pass + return str(syscall_id) + + def process_event(self, sample: perf.sample_event) -> None: + """Collect syscall events.""" + if not self.offline and sample.sample_pid == self.own_pid: + return + + name = str(sample.evsel) + # Per-syscall syscalls:sys_enter_* tracepoints expose '__syscall_nr' (or 'nr') + # and may have an unrelated syscall argument named 'id', whereas + # raw_syscalls:sys_enter (and legacy pre-2.6.35 syscalls:sys_enter) expose 'id'. + if name.startswith("evsel(syscalls:sys_enter_"): + syscall_id = getattr(sample, "__syscall_nr", -1) + if not (0 <= (syscall_id & ~0x40000000) <= 0xffff): + syscall_id = getattr(sample, "nr", -1) + elif name.startswith(("evsel(raw_syscalls:sys_enter", "evsel(syscalls:sys_enter")): + syscall_id = getattr(sample, "id", -1) + if not (0 <= (syscall_id & ~0x40000000) <= 0xffff): + syscall_id = getattr(sample, "__syscall_nr", -1) + if not (0 <= (syscall_id & ~0x40000000) <= 0xffff): + syscall_id = getattr(sample, "nr", -1) + else: + syscall_id = -1 + + skip = False + with self.lock: + if self.for_comm is not None: + is_execve = (0 <= (syscall_id & ~0x40000000) <= 0xffff and + self.syscall_name(syscall_id) in ("execve", "execveat")) + if is_execve: + self.comm_cache.pop(sample.sample_pid, None) + + comm = "Unknown" + if hasattr(self, 'session') and self.session: + # In offline perf.data mode, query session.find_thread() directly + # so PERF_RECORD_COMM updates after execve (e.g. perf -> sleep) + # are reflected immediately rather than returning a stale cached comm. + try: + proc = self.session.find_thread(sample.sample_pid, sample.sample_tid) + if proc: + comm = proc.comm() or "Unknown" + except TypeError: + pass + if comm != "Unknown" and not is_execve: + self.comm_cache[sample.sample_pid] = comm + elif sample.sample_pid in self.comm_cache: + comm = self.comm_cache[sample.sample_pid] + elif sample.sample_pid in self.comm_cache: + comm = self.comm_cache[sample.sample_pid] + else: + try: + with open(f"/proc/{sample.sample_pid}/comm", "r", + encoding="utf-8", errors="replace") as f: + comm = f.read().strip() + except OSError: + comm = "Unknown" + # Cache both matching and non-matching comms (including "Unknown" + # when a PID has exited or is inaccessible) so live system-wide + # tracing does not re-open /proc/<pid>/comm on every syscall. + # Do not cache during sys_enter(execve/execveat) since /proc/<pid>/comm + # still holds the pre-exec command name until the syscall completes. + if not is_execve: + self.comm_cache[sample.sample_pid] = comm + + if comm != self.for_comm: + skip = True + + is_enter = (name.startswith("evsel(raw_syscalls:sys_enter") or + name.startswith("evsel(syscalls:sys_enter")) + if not skip and is_enter and 0 <= (syscall_id & ~0x40000000) <= 0xffff: + self.syscalls[syscall_id] += 1 + + if self.offline and hasattr(sample, "sample_time"): + interval_ns = self.interval * (10 ** 9) + if self.last_print_time is None: + self.last_print_time = sample.sample_time + elif sample.sample_time - self.last_print_time >= interval_ns: + self.print_current_totals() + self.last_print_time = sample.sample_time + + def print_current_totals(self): + """Print current syscall totals.""" + self.printed = True + # Clear terminal + if not self.offline: + print("\x1b[2J\x1b[H", end="") + else: + print() + + with self.lock: + for_comm = self.for_comm + if for_comm is not None: + print(f"\nsyscall events for {for_comm}:\n") + else: + print("\nsyscall events:\n") + + print(f"{'event':40s} {'count':10s}") + print(f"{'-' * 40:40s} {'-' * 10:10s}") + + with self.lock: + current_syscalls = list(self.syscalls.items()) + self.syscalls.clear() + self.comm_cache.clear() + + current_syscalls.sort(key=lambda kv: (-kv[1], kv[0])) + + for syscall_id, val in current_syscalls: + print(f"{self.syscall_name(syscall_id):<40s} {val:10d}") + + def print_syscall_totals(self): + """Periodically print syscall totals.""" + while not self.stop_event.is_set(): + self.print_current_totals() + self.stop_event.wait(self.interval) + # Print final batch + self.print_current_totals() + + def start(self): + """Start the background thread.""" + self.thread.start() + + def stop(self): + """Stop the background thread.""" + self.stop_event.set() + self.thread.join() + + +def main(): + """Main function.""" + ap = argparse.ArgumentParser(description="System call top") + ap.add_argument("args", nargs="*", help="[comm] [interval] or [interval]") + ap.add_argument("-i", "--input", help="Input file name") + args = ap.parse_args() + + for_comm = None + default_interval = 3 + interval = default_interval + + if len(args.args) > 2: + print("Usage: python sctop.py [comm] [interval]") + sys.exit(1) + + if len(args.args) > 1: + for_comm = args.args[0] + try: + interval = int(args.args[1]) + except ValueError: + print(f"Invalid interval: {args.args[1]}") + sys.exit(1) + elif len(args.args) > 0: + try: + interval = int(args.args[0]) + except ValueError: + for_comm = args.args[0] + interval = default_interval + + analyzer = SCTopAnalyzer(for_comm, interval, offline=bool(args.input)) + session = None + + try: + if args.input: + session = perf.session(perf.data(args.input), sample=analyzer.process_event) + analyzer.session = session + analyzer.e_machine = getattr(session, "e_machine", None) + session.process_events() + else: + try: + live_session = LiveSession( + "raw_syscalls:sys_enter", sample_callback=analyzer.process_event + ) + except OSError: + live_session = LiveSession( + "syscalls:sys_enter_*", sample_callback=analyzer.process_event + ) + analyzer.start() + live_session.run() + except KeyboardInterrupt: + pass + except (OSError, IOError) as e: + print(f"Error: {e}", file=sys.stderr) + sys.exit(1) + finally: + if args.input: + if not analyzer.printed or analyzer.syscalls: + analyzer.print_current_totals() + # Break the reference cycle between perf.session and analyzer.process_event + # because perf.session lacks cyclic GC support (tp_traverse). + analyzer.session = None + session = None + elif analyzer.thread.is_alive(): + analyzer.stop() + + +if __name__ == "__main__": + main() diff --git a/tools/perf/python/stackcollapse.py b/tools/perf/python/stackcollapse.py new file mode 100755 index 000000000000..0e8a65969db3 --- /dev/null +++ b/tools/perf/python/stackcollapse.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +""" +stackcollapse.py - format perf samples with one line per distinct call stack + +This script's output has two space-separated fields. The first is a semicolon +separated stack including the program name (from the "comm" field) and the +function names from the call stack. The second is a count: + + swapper;start_kernel;rest_init;cpu_idle;default_idle;native_safe_halt 2 + +The file is sorted according to the first field. + +Ported from tools/perf/scripts/python/stackcollapse.py +""" +from __future__ import annotations + +import argparse +from collections import defaultdict +import os +import sys +import perf + + +class StackCollapseAnalyzer: + """Accumulates call stacks and prints them collapsed.""" + + def __init__(self, args: argparse.Namespace) -> None: + self.args = args + self.lines: dict[str, int] = defaultdict(int) + self.session: perf.session | None = None + + def tidy_function_name(self, sym: str, dso: str) -> str: + """Beautify function names based on options.""" + if sym is None: + sym = "[unknown]" + + sym = sym.replace(";", ":") + if self.args.tidy_java: + # Beautify Java signatures + sym = sym.replace("<", "") + sym = sym.replace(">", "") + if sym.startswith("L") and "/" in sym: + sym = sym[1:] + try: + sym = sym[:sym.index("(")] + except ValueError: + pass + + if self.args.annotate_kernel and dso == "[kernel.kallsyms]": + return sym + "_[k]" + return sym + + def process_event(self, sample: perf.sample_event) -> None: + """Collect call stack for each sample.""" + stack = [] + callchain = sample.callchain + if callchain is not None: + for node in callchain: + stack.append(self.tidy_function_name(node.symbol, node.dso)) + else: + # Fallback if no callchain + sym = (sample.symbol or '[unknown]') + dso = (sample.dso or '[unknown]') + stack.append(self.tidy_function_name(sym, dso)) + + if self.args.include_comm: + comm = "Unknown" + if self.session is not None: + try: + proc = self.session.find_thread( + sample.sample_pid, sample.sample_tid + ) + if proc: + proc_comm = proc.comm() + if proc_comm is not None: + comm = proc_comm + except TypeError: + pass + comm = str(comm).replace(" ", "_") + sep = "-" + if self.args.include_pid: + comm = f"{comm}{sep}{(sample.sample_pid or 0)}" + sep = "/" + if self.args.include_tid: + comm = f"{comm}{sep}{(sample.sample_tid or 0)}" + stack.append(comm) + + stack_string = ";".join(reversed(stack)) + self.lines[stack_string] += 1 + + def print_totals(self) -> None: + """Print sorted collapsed stacks.""" + try: + for stack in sorted(self.lines): + print(f"{stack} {self.lines[stack]}") + sys.stdout.flush() + except BrokenPipeError: + devnull = os.open(os.devnull, os.O_WRONLY) + os.dup2(devnull, sys.stdout.fileno()) + os.close(devnull) + + +def main(): + """Main function.""" + ap = argparse.ArgumentParser( + description="Format perf samples with one line per distinct call stack" + ) + ap.add_argument("-i", "--input", default="perf.data", help="Input file name") + ap.add_argument("--include-tid", action="store_true", help="include thread id in stack") + ap.add_argument("--include-pid", action="store_true", help="include process id in stack") + ap.add_argument("--no-comm", dest="include_comm", action="store_false", default=True, + help="do not separate stacks according to comm") + ap.add_argument("--tidy-java", action="store_true", help="beautify Java signatures") + ap.add_argument("--kernel", dest="annotate_kernel", action="store_true", + help="annotate kernel functions with _[k]") + + args = ap.parse_args() + + if args.include_tid and not args.include_comm: + print("requesting tid but not comm is invalid", file=sys.stderr) + sys.exit(1) + if args.include_pid and not args.include_comm: + print("requesting pid but not comm is invalid", file=sys.stderr) + sys.exit(1) + + analyzer = StackCollapseAnalyzer(args) + + try: + session = perf.session(perf.data(args.input), sample=analyzer.process_event) + analyzer.session = session + session.process_events() + except IOError as e: + print(f"Error: {e}", file=sys.stderr) + sys.exit(1) + except KeyboardInterrupt: + pass + finally: + analyzer.session = None + + analyzer.print_totals() + + +if __name__ == "__main__": + main() diff --git a/tools/perf/python/stat-cpi.py b/tools/perf/python/stat-cpi.py new file mode 100755 index 000000000000..b4ec87b07938 --- /dev/null +++ b/tools/perf/python/stat-cpi.py @@ -0,0 +1,231 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +"""Calculate CPI from perf stat data or live.""" +from __future__ import annotations + +import argparse +import os +import signal +import sys +import time +from typing import Any, Optional +import perf + +class StatCpiAnalyzer: + """Accumulates cycles and instructions and calculates CPI.""" + + def __init__(self, args: argparse.Namespace) -> None: + self.args = args + self.data: dict[str, float] = {} + self.prev_data: dict[str, tuple[int, int, int]] = {} + self.recorded_pairs: set[tuple[int, int]] = set() + + def get_key(self, event: str, cpu: int, thread: int) -> str: + """Get key for data dictionary.""" + return f"{event}-{cpu}-{thread}" + + def store_key(self, cpu: int, thread: int) -> None: + """Store CPU and thread IDs.""" + self.recorded_pairs.add((cpu, thread)) + + def store(self, event: str, cpu: int, thread: int, + counts: tuple[int, int, int], is_delta: bool = False, + raw_name: Optional[str] = None) -> None: + """Store counter values, computing difference from previous + absolute values if not already deltas.""" + self.store_key(cpu, thread) + key = self.get_key(event, cpu, thread) + prev_key = self.get_key(raw_name or event, cpu, thread) + + val, ena, run = counts + if is_delta: + # counts are already deltas + cur_val = val + cur_ena = ena + cur_run = run + else: + if prev_key in self.prev_data: + prev_val, prev_ena, prev_run = self.prev_data[prev_key] + cur_val = val - prev_val + cur_ena = ena - prev_ena + cur_run = run - prev_run + else: + cur_val = val + cur_ena = ena + cur_run = run + self.prev_data[prev_key] = counts # Store absolute value for next time + + # Scale each raw event's delta by its own multiplexing ratio before + # summing across PMUs (e.g. cpu_core and cpu_atom on hybrid systems) so + # enabled time from an idle PMU does not inflate the multiplier of an + # active PMU. + scaled_val = cur_val * (cur_ena / float(cur_run)) if cur_run > 0 else float(cur_val) + self.data[key] = self.data.get(key, 0.0) + scaled_val + + def get(self, event: str, cpu: int, thread: int) -> float: + """Get scaled counter value.""" + key = self.get_key(event, cpu, thread) + return self.data.get(key, 0.0) + + @staticmethod + def _classify_event(name: str) -> Optional[str]: + """Classify an event name as 'cycles' or 'instructions'.""" + ev = name[6:-1] if name.startswith("evsel(") and name.endswith(")") else name + ev = ev.split(":", 1)[0] + if "/" in ev: + parts = [p for p in ev.split("/") if p] + if len(parts) >= 2: + ev = parts[1] + if ev in ("cycles", "cpu-cycles"): + return "cycles" + if ev == "instructions": + return "instructions" + return None + + def process_stat_event(self, event: Any, name: Optional[str] = None) -> None: + """Process PERF_RECORD_STAT and PERF_RECORD_STAT_ROUND events.""" + if event.type == perf.RECORD_STAT: + if name: + event_name = self._classify_event(name) + if not event_name: + return + self.store(event_name, event.cpu, event.thread, + (event.val, event.ena, event.run), raw_name=name) + elif event.type == perf.RECORD_STAT_ROUND: + timestamp = getattr(event, "time", 0) + self.print_interval(timestamp) + self.data.clear() + self.recorded_pairs.clear() + + def print_interval(self, timestamp: int) -> None: + """Print CPI for the current interval.""" + for cpu, thread in sorted(self.recorded_pairs): + cyc = self.get("cycles", cpu, thread) + ins = self.get("instructions", cpu, thread) + cpi = 0.0 + if ins != 0: + cpi = cyc / float(ins) + t_sec = timestamp / 1000000000.0 + print(f"{t_sec:15f}: cpu {cpu}, thread {thread} -> cpi {cpi:f} ({cyc:.0f}/{ins:.0f})", + flush=True) + + def read_counters(self, evlist: Any) -> None: + """Read counters live.""" + for evsel in evlist: + name = str(evsel) + event_name = self._classify_event(name) + if not event_name: + continue + + for cpu in evsel.cpus(): + for thread in evsel.threads(): + try: + counts = evsel.read(cpu, thread) + self.store(event_name, cpu, thread, + (counts.val, counts.ena, counts.run), + is_delta=True, raw_name=name) + except OSError: + pass + + def run_file(self) -> None: + """Process events from file.""" + session: Optional[perf.session] = perf.session( + perf.data(self.args.input), stat=self.process_stat_event + ) + try: + assert session is not None + session.process_events() + finally: + session = None + + def _open_live_evlist(self) -> Any: + """Open evlist for live mode, falling back to user-space or process scope on EACCES.""" + threads = perf.thread_map(self.args.pid) if self.args.pid else None + candidates = [ + ("cycles,instructions", threads), + ("cycles:u,instructions:u", threads), + ] + if threads is None: + self_threads = perf.thread_map(os.getpid()) + candidates.append(("cycles,instructions", self_threads)) + candidates.append(("cycles:u,instructions:u", self_threads)) + + last_err: Optional[OSError] = None + for events, tmap in candidates: + evlist = None + try: + evlist = perf.parse_events(events, None, tmap) + for evsel in evlist: + evsel.read_format |= ( + perf.FORMAT_TOTAL_TIME_ENABLED | perf.FORMAT_TOTAL_TIME_RUNNING + ) + evlist.open() + evlist.enable() + return evlist + except PermissionError as e: + if evlist is not None: + evlist.close() + last_err = e + except OSError as e: + if evlist is not None: + evlist.close() + if e.errno == 13: + last_err = e + else: + raise + except BaseException: + if evlist is not None: + evlist.close() + raise + if last_err is not None: + raise last_err + raise RuntimeError("Failed to open events") + + def run_live(self) -> None: + """Read counters live.""" + def handle_signal(_signum: int, _frame: Any) -> None: + raise KeyboardInterrupt + + signal.signal(signal.SIGINT, signal.default_int_handler) + signal.signal(signal.SIGTERM, handle_signal) + + evlist = None + try: + try: + evlist = self._open_live_evlist() + except OSError as e: + print(f"Failed to open events: {e}", file=sys.stderr) + sys.exit(1) + + print("Live mode started. Press Ctrl+C to stop.", flush=True) + while True: + time.sleep(self.args.interval) + timestamp = time.time_ns() + self.read_counters(evlist) + self.print_interval(timestamp) + self.data.clear() + self.recorded_pairs.clear() + except KeyboardInterrupt: + print("\nStopped.", flush=True) + finally: + if evlist is not None: + evlist.close() + +def main() -> None: + """Main function.""" + ap = argparse.ArgumentParser(description="Calculate CPI from perf stat data or live") + ap.add_argument("-i", "--input", help="Input file name (enables file mode)") + ap.add_argument("-I", "--interval", type=float, default=1.0, + help="Interval in seconds for live mode") + ap.add_argument("-p", "--pid", type=int, + help="Monitor specific process ID in live mode") + args = ap.parse_args() + + analyzer = StatCpiAnalyzer(args) + if args.input: + analyzer.run_file() + else: + analyzer.run_live() + +if __name__ == "__main__": + main() diff --git a/tools/perf/python/syscall-counts-by-pid.py b/tools/perf/python/syscall-counts-by-pid.py new file mode 100755 index 000000000000..fc0ec1bf7ca1 --- /dev/null +++ b/tools/perf/python/syscall-counts-by-pid.py @@ -0,0 +1,112 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +""" +Displays system-wide system call totals, broken down by syscall. +If a [comm] arg is specified, only syscalls called by [comm] are displayed. +""" +from __future__ import annotations + +import argparse +from collections import defaultdict +from typing import (Dict, Tuple) +import perf + +syscalls: Dict[Tuple[str, int, int], int] = defaultdict(int) +for_comm = None +for_pid = None +session = None + + +def print_syscall_totals(): + """Print aggregated statistics.""" + if for_comm is not None: + print(f"\nsyscall events for {for_comm}:\n") + elif for_pid is not None: + print(f"\nsyscall events for PID {for_pid}:\n") + else: + print("\nsyscall events:\n") + + print(f"{'comm [pid]/syscalls':<40} {'count':>10}") + print("---------------------------------------- -----------") + + sorted_keys = sorted(syscalls.keys(), key=lambda k: (k[0], k[1], -syscalls[k], k[2])) + current_comm_pid = None + for comm, pid, sc_id in sorted_keys: + if current_comm_pid != (comm, pid): + print(f"\n{comm} [{pid}]") + current_comm_pid = (comm, pid) + e_machine = getattr(session, "e_machine", 0) or 0 + # Mask out the x86_64 x32 ABI bit (__X32_SYSCALL_BIT = 0x40000000) before + # resolving the syscall number in the architecture's syscall table. + raw_sc_id = sc_id & ~0x40000000 + if e_machine: + name = perf.syscall_name(raw_sc_id, e_machine) or str(sc_id) + else: + name = perf.syscall_name(raw_sc_id) or str(sc_id) + print(f" {name:<38} {syscalls[(comm, pid, sc_id)]:>10}") + + +def process_event(sample): + """Process a single sample event.""" + event_name = str(sample.evsel) + # Per-syscall syscalls:sys_enter_* tracepoints expose '__syscall_nr' (or 'nr') + # and may have an unrelated syscall argument named 'id', whereas + # raw_syscalls:sys_enter (and legacy pre-2.6.35 syscalls:sys_enter) expose 'id'. + if event_name.startswith("evsel(syscalls:sys_enter_"): + sc_id = getattr(sample, "__syscall_nr", None) + if sc_id is not None and not (0 <= (sc_id & ~0x40000000) <= 0xffff): + sc_id = None + if sc_id is None: + sc_id = getattr(sample, "nr", -1) + elif event_name.startswith(("evsel(raw_syscalls:sys_enter", "evsel(syscalls:sys_enter")): + sc_id = getattr(sample, "id", -1) + if not (0 <= (sc_id & ~0x40000000) <= 0xffff): + sc_id = getattr(sample, "__syscall_nr", -1) + if not (0 <= (sc_id & ~0x40000000) <= 0xffff): + sc_id = getattr(sample, "nr", -1) + else: + return + + # Mask out __X32_SYSCALL_BIT (0x40000000) when validating the syscall ID range. + if not (0 <= (sc_id & ~0x40000000) <= 0xffff): + return + + pid = sample.sample_pid + + if for_pid is not None and pid != for_pid: + return + + comm = "unknown" + try: + if session: + proc = session.find_thread(sample.sample_pid, sample.sample_tid) + if proc: + comm = proc.comm() or "unknown" + except (TypeError, AttributeError): + pass + + if for_comm and comm != for_comm: + return + syscalls[(comm, pid, sc_id)] += 1 + + +if __name__ == "__main__": + ap = argparse.ArgumentParser() + ap.add_argument("filter", nargs="?", help="COMM or PID to filter by") + ap.add_argument("-i", "--input", default="perf.data", help="Input file name") + args = ap.parse_args() + + if args.filter: + try: + for_pid = int(args.filter) + except ValueError: + for_comm = args.filter + + try: + session = perf.session(perf.data(args.input), sample=process_event) + session.process_events() + print_syscall_totals() + finally: + # Break the reference cycle between session and process_event (whose module + # globals reference session) since perf.session lacks cyclic GC (tp_traverse). + session = None diff --git a/tools/perf/python/syscall-counts.py b/tools/perf/python/syscall-counts.py new file mode 100755 index 000000000000..3ddb1bcc27e0 --- /dev/null +++ b/tools/perf/python/syscall-counts.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +""" +Displays system-wide system call totals, broken down by syscall. + +If a [comm] arg is specified, only syscalls called by [comm] are displayed. +""" +from __future__ import annotations + +import argparse +from collections import defaultdict +from typing import DefaultDict +import perf + +syscalls: DefaultDict[int, int] = defaultdict(int) +for_comm = None +session = None + + +def print_syscall_totals(): + """Print aggregated statistics.""" + if for_comm is not None: + print(f"\nsyscall events for {for_comm}:\n") + else: + print("\nsyscall events:\n") + + print(f"{'event':<40} {'count':>10}") + print("---------------------------------------- -----------") + + for sc_id, val in sorted(syscalls.items(), + key=lambda kv: (-kv[1], kv[0])): + e_machine = getattr(session, "e_machine", 0) or 0 + # Mask out the x86_64 x32 ABI bit (__X32_SYSCALL_BIT = 0x40000000) before + # resolving the syscall number in the architecture's syscall table. + raw_sc_id = sc_id & ~0x40000000 + if e_machine: + name = perf.syscall_name(raw_sc_id, e_machine) or str(sc_id) + else: + name = perf.syscall_name(raw_sc_id) or str(sc_id) + print(f"{name:<40} {val:>10}") + + +def process_event(sample): + """Process a single sample event.""" + event_name = str(sample.evsel) + # Per-syscall syscalls:sys_enter_* tracepoints expose '__syscall_nr' (or 'nr') + # and may have an unrelated syscall argument named 'id' (e.g. sys_enter_clock_gettime), + # whereas raw_syscalls:sys_enter (and legacy pre-2.6.35 syscalls:sys_enter) expose 'id'. + if event_name.startswith("evsel(syscalls:sys_enter_"): + sc_id = getattr(sample, "__syscall_nr", None) + if sc_id is not None and not (0 <= (sc_id & ~0x40000000) <= 0xffff): + sc_id = None + if sc_id is None: + sc_id = getattr(sample, "nr", -1) + elif event_name.startswith(("evsel(raw_syscalls:sys_enter", "evsel(syscalls:sys_enter")): + sc_id = getattr(sample, "id", -1) + if not (0 <= (sc_id & ~0x40000000) <= 0xffff): + sc_id = getattr(sample, "__syscall_nr", -1) + if not (0 <= (sc_id & ~0x40000000) <= 0xffff): + sc_id = getattr(sample, "nr", -1) + else: + return + + # Mask out __X32_SYSCALL_BIT (0x40000000) when validating the syscall ID range. + if not (0 <= (sc_id & ~0x40000000) <= 0xffff): + return + + comm = "unknown" + try: + if session: + proc = session.find_thread(sample.sample_pid, sample.sample_tid) + if proc: + comm = proc.comm() or "unknown" + except (TypeError, AttributeError): + pass + + if for_comm and comm != for_comm: + return + syscalls[sc_id] += 1 + + +if __name__ == "__main__": + ap = argparse.ArgumentParser() + ap.add_argument("comm", nargs="?", help="Only report syscalls for comm") + ap.add_argument("-i", "--input", default="perf.data", help="Input file name") + args = ap.parse_args() + for_comm = args.comm + try: + session = perf.session(perf.data(args.input), sample=process_event) + session.process_events() + print_syscall_totals() + finally: + # Break the reference cycle between session and process_event (whose module + # globals reference session) since perf.session lacks cyclic GC (tp_traverse). + session = None diff --git a/tools/perf/python/task-analyzer.py b/tools/perf/python/task-analyzer.py new file mode 100755 index 000000000000..fedba36b5ced --- /dev/null +++ b/tools/perf/python/task-analyzer.py @@ -0,0 +1,908 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +# task-analyzer.py - comprehensive perf tasks analysis +# Copyright (c) 2022, Hagen Paul Pfeifer <hagen@jauu.net> +# Licensed under the terms of the GNU GPL License version 2 +# +# Usage: +# +# perf record -e sched:sched_switch -a -- sleep 10 +# ./task-analyzer.py +# +"""Comprehensive perf tasks analysis.""" +from __future__ import annotations + +import argparse +from contextlib import contextmanager +import decimal + +from typing import List, Dict, Union + +def _median(numbers: List[decimal.Decimal]) -> decimal.Decimal: + """phython3 hat statistics module - we have nothing""" + n = len(numbers) + index = n // 2 + if n % 2: + return sorted(numbers)[index] + return sum(sorted(numbers)[index - 1 : index + 1]) / decimal.Decimal(2) + +def _mean(numbers: List[decimal.Decimal]) -> decimal.Decimal: + return sum(numbers) / decimal.Decimal(len(numbers)) + +import os +import string +import sys +from typing import Any, Optional +import perf + + +# Columns will have a static size to align everything properly +# Support of 116 days of active update with nano precision +LEN_SWITCHED_IN = len("9999999.999999999") +LEN_SWITCHED_OUT = len("9999999.999999999") +LEN_CPU = len("000") +LEN_PID = len("maxvalue") +LEN_TID = len("maxvalue") +LEN_COMM = len("max-comms-length") +LEN_RUNTIME = len("999999.999") +# Support of 3.45 hours of timespans +LEN_OUT_IN = len("99999999999.999") +LEN_OUT_OUT = len("99999999999.999") +LEN_IN_IN = len("99999999999.999") +LEN_IN_OUT = len("99999999999.999") + +class Timespans: + """Tracks elapsed time between occurrences of the same task.""" + def __init__(self, args: argparse.Namespace, time_unit: str) -> None: + self.args = args + self.time_unit = time_unit + self._last_start: Optional[decimal.Decimal] = None + self._last_finish: Optional[decimal.Decimal] = None + self.current = { + 'out_out': decimal.Decimal(-1), + 'in_out': decimal.Decimal(-1), + 'out_in': decimal.Decimal(-1), + 'in_in': decimal.Decimal(-1) + } + if args.summary_extended: + self._time_in: decimal.Decimal = decimal.Decimal(-1) + self.max_vals = { + 'out_in': decimal.Decimal(-1), + 'at': decimal.Decimal(-1), + 'in_out': decimal.Decimal(-1), + 'in_in': decimal.Decimal(-1), + 'out_out': decimal.Decimal(-1) + } + + def feed(self, task: 'Task') -> None: + """Calculate timespans from chronological task occurrences.""" + if not self._last_finish: + self._last_start = task.time_in(self.time_unit) + self._last_finish = task.time_out(self.time_unit) + return + assert self._last_start is not None + assert self._last_finish is not None + self._time_in = task.time_in() + time_in = task.time_in(self.time_unit) + time_out = task.time_out(self.time_unit) + self.current['in_in'] = time_in - self._last_start + self.current['out_in'] = time_in - self._last_finish + self.current['in_out'] = time_out - self._last_start + self.current['out_out'] = time_out - self._last_finish + if self.args.summary_extended: + self.update_max_entries() + self._last_finish = task.time_out(self.time_unit) + self._last_start = task.time_in(self.time_unit) + + def update_max_entries(self) -> None: + """Update maximum timespans.""" + self.max_vals['in_in'] = max(self.max_vals['in_in'], self.current['in_in']) + self.max_vals['out_out'] = max(self.max_vals['out_out'], self.current['out_out']) + self.max_vals['in_out'] = max(self.max_vals['in_out'], self.current['in_out']) + if self.current['out_in'] > self.max_vals['out_in']: + self.max_vals['out_in'] = self.current['out_in'] + self.max_vals['at'] = self._time_in + +class Task: + """Handles information of a given task.""" + def __init__(self, task_id: str, tid: int, cpu: int, comm: str) -> None: + self.id = task_id + self.tid = tid + self.cpu = cpu + self.comm = comm + self.pid: Optional[int] = None + self._time_in: Optional[decimal.Decimal] = None + self._time_out: Optional[decimal.Decimal] = None + + def schedule_in_at(self, time_ns: int) -> None: + """Set schedule in time.""" + self._time_in = decimal.Decimal(time_ns) / decimal.Decimal(1e9) + + def schedule_out_at(self, time_ns: int) -> None: + """Set schedule out time.""" + self._time_out = decimal.Decimal(time_ns) / decimal.Decimal(1e9) + + def time_out(self, unit: str = "s") -> decimal.Decimal: + """Return schedule out time.""" + factor = TaskAnalyzer.time_uniter(unit) + return self._time_out * decimal.Decimal(factor) if self._time_out else decimal.Decimal(0) + + def time_in(self, unit: str = "s") -> decimal.Decimal: + """Return schedule in time.""" + factor = TaskAnalyzer.time_uniter(unit) + return self._time_in * decimal.Decimal(factor) if self._time_in else decimal.Decimal(0) + + def runtime(self, unit: str = "us") -> decimal.Decimal: + """Return runtime.""" + factor = TaskAnalyzer.time_uniter(unit) + if self._time_out is not None and self._time_in is not None: + return (self._time_out - self._time_in) * decimal.Decimal(factor) + return decimal.Decimal(0) + + def update_pid(self, pid: int) -> None: + """Update PID.""" + self.pid = pid + +class Summary: + """ + Primary instance for calculating the summary output. Processes the whole trace to + find and memorize relevant data such as mean, max et cetera. This instance handles + dynamic alignment aspects for summary output. + """ + + def __init__(self, analyzer): + self.analyzer = analyzer + self.args = analyzer.args + self.db = analyzer.db + self.time_unit = analyzer.time_unit + self.fd_sum = analyzer.fd_sum + self._body = [] + + class AlignmentHelper: + """ + Used to calculated the alignment for the output of the summary. + """ + def __init__(self, pid, tid, comm, runs, acc, mean, + median, min_val, max_val, max_at): + self.pid = pid + self.tid = tid + self.comm = comm + self.runs = runs + self.acc = acc + self.mean = mean + self.median = median + self.min = min_val + self.max = max_val + self.max_at = max_at + self.out_in = None + self.inter_at = None + self.out_out = None + self.in_in = None + self.in_out = None + + def _print_header(self): + ''' + Output is aligned using the formatted column widths in self.db. + ''' + len_tasks = max( + len("Task Information"), + sum(self.db["task_info"].values()) + len(self.db["task_info"]) - 1, + ) + fmt = "{{:^{}}}".format(len_tasks) + fmt += " {{:^{}}}".format( + sum(self.db["runtime_info"].values()) + len(self.db["runtime_info"]) - 1 + ) + _header = ("Task Information", "Runtime Information") + + if self.args.summary_extended: + fmt += " {{:^{}}}".format( + sum(self.db["inter_times"].values()) + len(self.db["inter_times"]) - 1 + ) + _header += ("Max Inter Task Times",) + self.fd_sum.write(fmt.format(*_header) + "\n") + + def _column_titles(self): + """ + Cells are being processed and displayed in different way so an alignment adjust + is implemented depeding on the choice of the timeunit. The positions of the max + values are being displayed in grey. Thus in their format two additional {}, + are placed for color set and reset. + """ + separator, fix_csv_align = self.analyzer.prepare_fmt_sep(is_summary=True) + fmt = "{{:>{}}}".format(self.db["task_info"]["pid"] * fix_csv_align) + fmt += "{}{{:>{}}}".format(separator, self.db["task_info"]["tid"] * fix_csv_align) + fmt += "{}{{:>{}}}".format(separator, self.db["task_info"]["comm"] * fix_csv_align) + fmt += "{}{{:>{}}}".format(separator, self.db["runtime_info"]["runs"] * fix_csv_align) + fmt += "{}{{:>{}}}".format(separator, self.db["runtime_info"]["acc"] * fix_csv_align) + fmt += "{}{{:>{}}}".format(separator, self.db["runtime_info"]["mean"] * fix_csv_align) + fmt += "{}{{:>{}}}".format( + separator, self.db["runtime_info"]["median"] * fix_csv_align + ) + fmt += "{}{{:>{}}}".format( + separator, self.db["runtime_info"]["min"] * fix_csv_align + ) + fmt += "{}{{:>{}}}".format( + separator, self.db["runtime_info"]["max"] * fix_csv_align + ) + fmt += "{}{{}}{{:>{}}}{{}}".format( + separator, self.db["runtime_info"]["max_at"] * fix_csv_align + ) + + grey = "" if self.args.csv_summary else TaskAnalyzer.COLORS["grey"] + reset = "" if self.args.csv_summary else TaskAnalyzer.COLORS["reset"] + column_titles = ("PID", "TID", "Comm") + column_titles += ("Runs", "Accumulated", "Mean", "Median", "Min", "Max") + column_titles += (grey, "Max At", reset) + + if self.args.summary_extended: + fmt += "{}{{:>{}}}".format( + separator, + self.db["inter_times"]["out_in"] * fix_csv_align + ) + fmt += "{}{{}}{{:>{}}}{{}}".format( + separator, + self.db["inter_times"]["inter_at"] * fix_csv_align + ) + fmt += "{}{{:>{}}}".format( + separator, + self.db["inter_times"]["out_out"] * fix_csv_align + ) + fmt += "{}{{:>{}}}".format( + separator, + self.db["inter_times"]["in_in"] * fix_csv_align + ) + fmt += "{}{{:>{}}}".format( + separator, + self.db["inter_times"]["in_out"] * fix_csv_align + ) + + column_titles += ( + "Out-In", grey, "Max At", + reset, "Out-Out", "In-In", "In-Out" + ) + + self.fd_sum.write(fmt.format(*column_titles) + "\n") + + def _task_stats(self): + """calculates the stats of every task and constructs the printable summary""" + grey = "" if self.args.csv_summary else TaskAnalyzer.COLORS["grey"] + reset = "" if self.args.csv_summary else TaskAnalyzer.COLORS["reset"] + for tid in sorted(self.db["tid"]): + color_one_sample = grey + color_reset = reset + no_executed = 0 + runtimes = [] + time_in = [] + timespans = Timespans(self.args, self.time_unit) + for task in self.db["tid"][tid]: + pid = task.pid + comm = task.comm + no_executed += 1 + runtimes.append(task.runtime(self.time_unit)) + time_in.append(task.time_in()) + timespans.feed(task) + if len(runtimes) > 1: + color_one_sample = "" + color_reset = "" + time_max = max(runtimes) + time_min = min(runtimes) + max_at = time_in[runtimes.index(max(runtimes))] + + # The size of the decimal after sum,mean and median varies, thus we cut + # the decimal number, by rounding it. It has no impact on the output, + # because we have a precision of the decimal points at the output. + time_sum = round(sum(runtimes), 3) + time_mean = round(_mean(runtimes), 3) + time_median = round(_median(runtimes), 3) + + align_helper = self.AlignmentHelper(pid, tid, comm, no_executed, time_sum, + time_mean, time_median, time_min, time_max, max_at) + self._body.append([ + pid, tid, comm, no_executed, time_sum, color_one_sample, + time_mean, time_median, time_min, time_max, + grey, max_at, + reset, color_reset + ]) + if self.args.summary_extended: + self._body[-1].extend([timespans.max_vals['out_in'], + grey, timespans.max_vals['at'], + reset, timespans.max_vals['out_out'], + timespans.max_vals['in_in'], + timespans.max_vals['in_out']]) + align_helper.out_in = timespans.max_vals['out_in'] + align_helper.inter_at = timespans.max_vals['at'] + align_helper.out_out = timespans.max_vals['out_out'] + align_helper.in_in = timespans.max_vals['in_in'] + align_helper.in_out = timespans.max_vals['in_out'] + self._calc_alignments_summary(align_helper) + + def _format_stats(self): + separator, fix_csv_align = self.analyzer.prepare_fmt_sep(is_summary=True) + decimal_precision, time_precision = self.analyzer.prepare_fmt_precision() + len_pid = self.db["task_info"]["pid"] * fix_csv_align + len_tid = self.db["task_info"]["tid"] * fix_csv_align + len_comm = self.db["task_info"]["comm"] * fix_csv_align + len_runs = self.db["runtime_info"]["runs"] * fix_csv_align + len_acc = self.db["runtime_info"]["acc"] * fix_csv_align + len_mean = self.db["runtime_info"]["mean"] * fix_csv_align + len_median = self.db["runtime_info"]["median"] * fix_csv_align + len_min = self.db["runtime_info"]["min"] * fix_csv_align + len_max = self.db["runtime_info"]["max"] * fix_csv_align + len_max_at = self.db["runtime_info"]["max_at"] * fix_csv_align + if self.args.summary_extended: + len_out_in = self.db["inter_times"]["out_in"] * fix_csv_align + len_inter_at = self.db["inter_times"]["inter_at"] * fix_csv_align + len_out_out = self.db["inter_times"]["out_out"] * fix_csv_align + len_in_in = self.db["inter_times"]["in_in"] * fix_csv_align + len_in_out = self.db["inter_times"]["in_out"] * fix_csv_align + + fmt = "{{:{}d}}".format(len_pid) + fmt += "{}{{:{}d}}".format(separator, len_tid) + fmt += "{}{{:>{}}}".format(separator, len_comm) + fmt += "{}{{:{}d}}".format(separator, len_runs) + fmt += "{}{{:{}.{}f}}".format(separator, len_acc, time_precision) + fmt += "{}{{}}{{:{}.{}f}}".format(separator, len_mean, time_precision) + fmt += "{}{{:{}.{}f}}".format(separator, len_median, time_precision) + fmt += "{}{{:{}.{}f}}".format(separator, len_min, time_precision) + fmt += "{}{{:{}.{}f}}".format(separator, len_max, time_precision) + fmt += "{}{{}}{{:{}.{}f}}{{}}{{}}".format( + separator, len_max_at, decimal_precision + ) + if self.args.summary_extended: + fmt += "{}{{:{}.{}f}}".format(separator, len_out_in, time_precision) + fmt += "{}{{}}{{:{}.{}f}}{{}}".format( + separator, len_inter_at, decimal_precision + ) + fmt += "{}{{:{}.{}f}}".format(separator, len_out_out, time_precision) + fmt += "{}{{:{}.{}f}}".format(separator, len_in_in, time_precision) + fmt += "{}{{:{}.{}f}}".format(separator, len_in_out, time_precision) + return fmt + + def _calc_alignments_summary(self, align_helper): + # Measure the formatted string widths (e.g. f"{val:.{decimal_precision}f}" + # and f"{val:.{time_precision}f}") rather than str(val), because Decimal's + # default string representation may include unrounded fractional digits or + # omit trailing zeros that _format_stats() renders with fixed precision. + decimal_precision, time_precision = self.analyzer.prepare_fmt_precision() + for key in self.db["task_info"]: + val_len = len(str(getattr(align_helper, key))) + if val_len > self.db["task_info"][key]: + self.db["task_info"][key] = val_len + for key in self.db["runtime_info"]: + val = getattr(align_helper, key) + if key == "runs": + val_len = len(str(val)) + elif key == "max_at": + val_len = len(f"{val:.{decimal_precision}f}") + else: + val_len = len(f"{val:.{time_precision}f}") + if val_len > self.db["runtime_info"][key]: + self.db["runtime_info"][key] = val_len + if self.args.summary_extended: + for key in self.db["inter_times"]: + val = getattr(align_helper, key) + if key == "inter_at": + val_len = len(f"{val:.{decimal_precision}f}") + else: + val_len = len(f"{val:.{time_precision}f}") + if val_len > self.db["inter_times"][key]: + self.db["inter_times"][key] = val_len + + def print(self): + self._task_stats() + fmt = self._format_stats() + + if not self.args.csv_summary: + print("\nSummary", file=self.fd_sum) + self._print_header() + self._column_titles() + for i in range(len(self._body)): + self.fd_sum.write(fmt.format(*tuple(self._body[i])) + "\n") + + +class TaskAnalyzer: + + """Main class for task analysis.""" + + COLORS = { + "grey": "\033[90m", + "red": "\033[91m", + "green": "\033[92m", + "yellow": "\033[93m", + "blue": "\033[94m", + "violet": "\033[95m", + "reset": "\033[0m", + } + + def __init__(self, args: argparse.Namespace) -> None: + self.args = args + self.db: Dict[str, Any] = {} + self.session: Optional[perf.session] = None + self._tgid_cache: Dict[int, int] = {} + self.time_unit = "us" + if args.ns: + self.time_unit = "ns" + elif args.ms: + self.time_unit = "ms" + self._init_db() + self._check_color() + self.fd_task = sys.stdout + self.fd_sum = sys.stdout + + @contextmanager + def open_output(self, filename: str, default: Any): + """Context manager for file or stdout.""" + if filename: + with open(filename, "w", encoding="utf-8") as f: + yield f + else: + yield default + + def _init_db(self) -> None: + self.db["running"] = {} + self.db["tid"] = {} + self.db["global"] = [] + if (self.args.summary or self.args.summary_extended or + self.args.summary_only or self.args.csv_summary): + self.db["task_info"] = {} + self.db["runtime_info"] = {} + self.db["task_info"]["pid"] = len("PID") + self.db["task_info"]["tid"] = len("TID") + self.db["task_info"]["comm"] = len("Comm") + self.db["runtime_info"]["runs"] = len("Runs") + self.db["runtime_info"]["acc"] = len("Accumulated") + self.db["runtime_info"]["max"] = len("Max") + self.db["runtime_info"]["max_at"] = len("Max At") + self.db["runtime_info"]["min"] = len("Min") + self.db["runtime_info"]["mean"] = len("Mean") + self.db["runtime_info"]["median"] = len("Median") + if self.args.summary_extended: + self.db["inter_times"] = {} + self.db["inter_times"]["out_in"] = len("Out-In") + self.db["inter_times"]["inter_at"] = len("Max At") + self.db["inter_times"]["out_out"] = len("Out-Out") + self.db["inter_times"]["in_in"] = len("In-In") + self.db["inter_times"]["in_out"] = len("In-Out") + + def _check_color(self) -> None: + """Check if color should be enabled.""" + if self.args.csv: + TaskAnalyzer.COLORS = {k: "" for k in TaskAnalyzer.COLORS} + return + if sys.stdout.isatty() and self.args.stdio_color != "never": + return + if self.args.stdio_color == "always": + return + TaskAnalyzer.COLORS = {k: "" for k in TaskAnalyzer.COLORS} + + @staticmethod + def time_uniter(unit: str) -> float: + """Return time unit factor.""" + picker = {"s": 1, "ms": 1e3, "us": 1e6, "ns": 1e9} + return picker[unit] + + def _task_id(self, pid: int, cpu: int) -> str: + return f"{pid}-{cpu}" + + def _filter_non_printable(self, unfiltered: Union[str, bytearray]) -> str: + # Strip non-printable characters, whitespace control characters + # (\r\n\t\x0b\x0c), quotes (", '), and ';' because --csv and + # --csv-summary emit unquoted ';'-delimited columns. Also replace a + # leading '=', '+', '-', or '@' with '_' to mitigate CSV formula / DDE + # injection attacks if an attacker-controlled process comm is opened in + # a spreadsheet program. + if isinstance(unfiltered, (bytearray, bytes)): + unfiltered = unfiltered.decode('utf-8', 'ignore') + filtered = "" + for char in unfiltered: + if char in string.printable and char not in "\r\n\t\x0b\x0c;\"'": + filtered += char + stripped = filtered.lstrip() + if stripped and stripped[0] in "=+-@": + filtered = "_" + stripped[1:] + return filtered + + def prepare_fmt_precision(self) -> tuple[int, int]: + if self.args.ns: + return 9, 0 + return 6, 3 + + def prepare_fmt_sep(self, is_summary: bool = False) -> tuple[str, int]: + # fix_csv_align multiplies format field widths: 0 in CSV mode collapses + # {:>N} padding to {:>0} (no extra spaces around ';'), while 1 in + # standard text mode preserves fixed-width column alignment. + if (is_summary and self.args.csv_summary) or (not is_summary and self.args.csv): + return ";", 0 + return " ", 1 + + def _fmt_header(self) -> str: + separator, fix_csv_align = self.prepare_fmt_sep() + fmt = f"{{:>{LEN_SWITCHED_IN*fix_csv_align}}}" + fmt += f"{separator}{{:>{LEN_SWITCHED_OUT*fix_csv_align}}}" + fmt += f"{separator}{{:>{LEN_CPU*fix_csv_align}}}" + fmt += f"{separator}{{:>{LEN_PID*fix_csv_align}}}" + fmt += f"{separator}{{:>{LEN_TID*fix_csv_align}}}" + fmt += f"{separator}{{:>{LEN_COMM*fix_csv_align}}}" + fmt += f"{separator}{{:>{LEN_RUNTIME*fix_csv_align}}}" + fmt += f"{separator}{{:>{LEN_OUT_IN*fix_csv_align}}}" + if self.args.extended_times: + fmt += f"{separator}{{:>{LEN_OUT_OUT*fix_csv_align}}}" + fmt += f"{separator}{{:>{LEN_IN_IN*fix_csv_align}}}" + fmt += f"{separator}{{:>{LEN_IN_OUT*fix_csv_align}}}" + return fmt + + def _fmt_body(self) -> str: + separator, fix_csv_align = self.prepare_fmt_sep() + decimal_precision, time_precision = self.prepare_fmt_precision() + fmt = f"{{}}{{:{LEN_SWITCHED_IN*fix_csv_align}.{decimal_precision}f}}" + fmt += f"{separator}{{:{LEN_SWITCHED_OUT*fix_csv_align}.{decimal_precision}f}}" + fmt += f"{separator}{{:{LEN_CPU*fix_csv_align}d}}" + fmt += f"{separator}{{:{LEN_PID*fix_csv_align}d}}" + fmt += f"{separator}{{}}{{:{LEN_TID*fix_csv_align}d}}{{}}" + fmt += f"{separator}{{}}{{:>{LEN_COMM*fix_csv_align}}}" + fmt += f"{separator}{{:{LEN_RUNTIME*fix_csv_align}.{time_precision}f}}" + if self.args.extended_times: + fmt += f"{separator}{{:{LEN_OUT_IN*fix_csv_align}.{time_precision}f}}" + fmt += f"{separator}{{:{LEN_OUT_OUT*fix_csv_align}.{time_precision}f}}" + fmt += f"{separator}{{:{LEN_IN_IN*fix_csv_align}.{time_precision}f}}" + fmt += f"{separator}{{:{LEN_IN_OUT*fix_csv_align}.{time_precision}f}}{{}}" + else: + fmt += f"{separator}{{:{LEN_OUT_IN*fix_csv_align}.{time_precision}f}}{{}}" + return fmt + + def _print_header(self) -> None: + fmt = self._fmt_header() + header = ["Switched-In", "Switched-Out", "CPU", "PID", "TID", "Comm", + "Runtime", "Time Out-In"] + if self.args.extended_times: + header += ["Time Out-Out", "Time In-In", "Time In-Out"] + self.fd_task.write(fmt.format(*header) + "\n") + + def _print_task_finish(self, task: Task) -> None: + c_row_set = "" + c_row_reset = "" + out_in: Any = -1 + out_out: Any = -1 + in_in: Any = -1 + in_out: Any = -1 + fmt = self._fmt_body() + + if str(task.tid) in self.args.highlight_tasks_map: + c_row_set = TaskAnalyzer.COLORS.get(self.args.highlight_tasks_map[str(task.tid)], '') + c_row_reset = TaskAnalyzer.COLORS["reset"] + if task.comm in self.args.highlight_tasks_map: + c_row_set = TaskAnalyzer.COLORS.get(self.args.highlight_tasks_map[task.comm], '') + c_row_reset = TaskAnalyzer.COLORS["reset"] + + c_tid_set = "" + c_tid_reset = "" + if task.pid == task.tid: + c_tid_set = TaskAnalyzer.COLORS["grey"] + c_tid_reset = TaskAnalyzer.COLORS["reset"] + + if task.tid in self.db["tid"]: + last_tid_task = self.db["tid"][task.tid][-1] + timespan_gap_tid = Timespans(self.args, self.time_unit) + timespan_gap_tid.feed(last_tid_task) + timespan_gap_tid.feed(task) + out_in = timespan_gap_tid.current['out_in'] + out_out = timespan_gap_tid.current['out_out'] + in_in = timespan_gap_tid.current['in_in'] + in_out = timespan_gap_tid.current['in_out'] + + if self.args.extended_times: + line_out = fmt.format(c_row_set, task.time_in(), task.time_out(), task.cpu, + task.pid, c_tid_set, task.tid, c_tid_reset, c_row_set, task.comm, + task.runtime(self.time_unit), out_in, out_out, in_in, in_out, + c_row_reset) + "\n" + else: + line_out = fmt.format(c_row_set, task.time_in(), task.time_out(), task.cpu, + task.pid, c_tid_set, task.tid, c_tid_reset, c_row_set, task.comm, + task.runtime(self.time_unit), out_in, c_row_reset) + "\n" + self.fd_task.write(line_out) + + def _record_cleanup(self, _list: list[Any]) -> list[Any]: + need_summary = (self.args.summary or self.args.summary_extended or + self.args.summary_only or self.args.csv_summary) + if not need_summary and len(_list) > 1: + return _list[len(_list) - 1:] + return _list + + def _record_by_tid(self, task: Task) -> None: + tid = task.tid + if tid not in self.db["tid"]: + self.db["tid"][tid] = [] + self.db["tid"][tid].append(task) + self.db["tid"][tid] = self._record_cleanup(self.db["tid"][tid]) + + def _record_global(self, task: Task) -> None: + self.db["global"].append(task) + self.db["global"] = self._record_cleanup(self.db["global"]) + + def _handle_task_finish(self, tid: int, cpu: int, time_ns: int, pid: int) -> None: + if tid == 0 and (not self.args.tid_renames or 0 not in self.args.tid_renames): + return + _id = self._task_id(tid, cpu) + if _id not in self.db["running"]: + return + task = self.db["running"][_id] + task.schedule_out_at(time_ns) + task.update_pid(pid) + del self.db["running"][_id] + + if not self._limit_filtered(tid, pid, task.comm): + if not self.args.summary_only: + self._print_task_finish(task) + self._record_by_tid(task) + self._record_global(task) + + def _handle_task_start(self, tid: int, cpu: int, comm: str, time_ns: int) -> None: + if tid == 0 and (not self.args.tid_renames or 0 not in self.args.tid_renames): + return + if tid in self.args.tid_renames: + comm = self._filter_non_printable(self.args.tid_renames[tid]) + _id = self._task_id(tid, cpu) + if _id in self.db["running"]: + return + task = Task(_id, tid, cpu, comm) + task.schedule_in_at(time_ns) + self.db["running"][_id] = task + + def _limit_filtered(self, tid: int, pid: int, comm: str) -> bool: + """Filter tasks based on CLI arguments.""" + match_filter = False + if self.args.filter_tasks: + if (str(tid) in self.args.filter_tasks or + str(pid) in self.args.filter_tasks or + comm in self.args.filter_tasks): + match_filter = True + + match_limit = False + if self.args.limit_to_tasks: + if (str(tid) in self.args.limit_to_tasks or + str(pid) in self.args.limit_to_tasks or + comm in self.args.limit_to_tasks): + match_limit = True + + if self.args.filter_tasks and match_filter: + return True + if self.args.limit_to_tasks and not match_limit: + return True + return False + + def _is_within_timelimit(self, time_ns: int) -> bool: + if not self.args.time_limit: + return True + time_s = decimal.Decimal(time_ns) / decimal.Decimal(1e9) + bounds = self.args.time_limit.split(":") + lower_bound = bounds[0] if len(bounds) > 0 else "" + upper_bound = bounds[1] if len(bounds) > 1 else "" + if lower_bound and time_s < decimal.Decimal(lower_bound): + return False + if upper_bound and time_s > decimal.Decimal(upper_bound): + return False + return True + + def process_event(self, sample: perf.sample_event) -> None: + """Process sched:sched_switch events.""" + if "sched:sched_switch" not in str(sample.evsel): + return + + time_ns = sample.sample_time + if not self._is_within_timelimit(time_ns): + return + + # Access tracepoint fields directly from sample object + try: + prev_pid = sample.prev_pid + next_pid = sample.next_pid + next_comm = sample.next_comm + common_cpu = sample.sample_cpu + except AttributeError: + if not self.db.get("warned_missing_fields"): + self.db["warned_missing_fields"] = True + print("Warning: sched:sched_switch sample missing tracepoint fields " + "(is libtraceevent support enabled?).", file=sys.stderr) + return + + next_comm = self._filter_non_printable(next_comm) + + # Task finish for previous task. Resolve TGID (process ID) from TID: + # in file mode we query perf.session.find_thread() (checking thread.pid > 0 + # so an unresolved -1 does not overwrite the prev_pid fallback), whereas in + # live mode (self.session is None) we read /proc/<tid>/status and cache + # the lookup in _tgid_cache. + prev_tgid = prev_pid # Fallback + if prev_pid != 0: + if self.session: + try: + thread = self.session.find_thread(-1, prev_pid) + if thread and thread.pid > 0: + prev_tgid = thread.pid + except (OSError, ValueError, KeyError, RuntimeError, TypeError, AttributeError): + pass + elif prev_pid in self._tgid_cache: + prev_tgid = self._tgid_cache[prev_pid] + else: + if len(self._tgid_cache) >= 4096: + self._tgid_cache.pop(next(iter(self._tgid_cache))) + self._tgid_cache[prev_pid] = prev_tgid + try: + with open(f"/proc/{prev_pid}/status", encoding="utf-8") as f: + for line in f: + if line.startswith("Tgid:"): + prev_tgid = int(line.split()[1]) + self._tgid_cache[prev_pid] = prev_tgid + break + except (OSError, ValueError, KeyError, RuntimeError): + pass + self._handle_task_finish(prev_pid, common_cpu, time_ns, prev_tgid) + # Task start for next task + self._handle_task_start(next_pid, common_cpu, next_comm, time_ns) + + def print_summary(self) -> None: + """Calculate and print summary.""" + need_summary = (self.args.summary or self.args.summary_extended or + self.args.summary_only or self.args.csv_summary) + if not need_summary: + return + + Summary(self).print() + + def _run_file(self) -> None: + if not self.args.summary_only: + self._print_header() + + session = perf.session(perf.data(self.args.input), sample=self.process_event) + self.session = session + try: + session.process_events() + except KeyboardInterrupt: + pass + finally: + # Break the reference cycle between self.session and the bound + # self.process_event callback so the C perf.session object is freed. + self.session = None + + if not self.db["global"]: + print(f"Warning: No sched:sched_switch trace events found in '{self.args.input}'.", + file=sys.stderr) + + self.print_summary() + + def _run_live(self) -> None: + if not self.args.summary_only: + self._print_header() + + cpus = perf.cpu_map() + threads = perf.thread_map(-1) + evlist = perf.parse_events("sched:sched_switch", cpus, threads) + evlist.config() + + evlist.open() + evlist.mmap() + evlist.enable() + + # Per-CPU perf ring buffers are drained sequentially in a loop, so events + # read from CPU N may have timestamps slightly earlier than events just + # read from CPU 0. Buffer events in pending_events and only dispatch those + # older than a 50 ms watermark behind max_ts so cross-CPU migrations and + # switches are processed in global timestamp order. + pending_events: list[perf.sample_event] = [] + print("Live mode started. Press Ctrl+C to stop.", file=sys.stderr) + try: + while True: + try: + evlist.poll(timeout=100) + except InterruptedError: + continue + for cpu in cpus: + while True: + event = evlist.read_on_cpu(cpu) + if not event: + break + if not isinstance(event, perf.sample_event): + continue + pending_events.append(event) + if pending_events: + pending_events.sort( + key=lambda e: getattr(e, 'sample_time', getattr(e, 'time', 0)) + ) + max_ts = getattr( + pending_events[-1], 'sample_time', getattr(pending_events[-1], 'time', 0) + ) + cutoff_ts = max_ts - 50_000_000 + ready_idx = 0 + for event in pending_events: + ev_ts = getattr(event, 'sample_time', getattr(event, 'time', 0)) + if ev_ts <= cutoff_ts: + self.process_event(event) + ready_idx += 1 + else: + break + del pending_events[:ready_idx] + except KeyboardInterrupt: + print("\nStopping live mode...", file=sys.stderr) + finally: + pending_events.sort(key=lambda e: getattr(e, 'sample_time', getattr(e, 'time', 0))) + for event in pending_events: + self.process_event(event) + evlist.close() + self.print_summary() + + def run(self) -> None: + """Run the session.""" + is_live = (self.args.live or + (not os.path.exists(self.args.input) and self.args.input == "perf.data")) + with self.open_output(self.args.csv, sys.stdout) as fd_task: + if self.args.csv and self.args.csv == self.args.csv_summary: + self.fd_task = fd_task + self.fd_sum = fd_task + if is_live: + self._run_live() + else: + self._run_file() + else: + with self.open_output(self.args.csv_summary, sys.stdout) as fd_sum: + self.fd_task = fd_task + self.fd_sum = fd_sum + if is_live: + self._run_live() + else: + self._run_file() + +def main() -> None: + """Main function.""" + parser = argparse.ArgumentParser(description="Analyze tasks behavior") + parser.add_argument("-i", "--input", default="perf.data", help="Input file name") + parser.add_argument("--time-limit", default="", help="print tasks only in time window") + parser.add_argument("--summary", action="store_true", + help="print additional runtime information") + parser.add_argument("--summary-only", action="store_true", + help="print only summary without traces") + parser.add_argument("--summary-extended", action="store_true", + help="print extended summary") + parser.add_argument("--ns", action="store_true", help="show timestamps in nanoseconds") + parser.add_argument("--ms", action="store_true", help="show timestamps in milliseconds") + parser.add_argument("--live", action="store_true", + help="force live mode (ignores -i/perf.data)") + parser.add_argument("--extended-times", action="store_true", + help="Show elapsed times between schedule in/out") + parser.add_argument("--filter-tasks", default="", help="filter tasks by tid, pid or comm") + parser.add_argument("--limit-to-tasks", default="", help="limit output to selected tasks") + parser.add_argument("--highlight-tasks", default="", help="colorize special tasks") + parser.add_argument("--rename-comms-by-tids", default="", help="rename task names by using tid") + parser.add_argument("--stdio-color", default="auto", choices=["always", "never", "auto"], + help="configure color output") + parser.add_argument("--csv", default="", help="Write trace to file") + parser.add_argument("--csv-summary", default="", help="Write summary to file") + + args = parser.parse_args() + args.tid_renames = {} + args.highlight_tasks_map = {} + args.filter_tasks = args.filter_tasks.split(",") if args.filter_tasks else [] + args.limit_to_tasks = args.limit_to_tasks.split(",") if args.limit_to_tasks else [] + + if args.rename_comms_by_tids: + for item in args.rename_comms_by_tids.split(","): + try: + tid, name = item.split(":", 1) + args.tid_renames[int(tid)] = name + except ValueError: + print(f"Error: Invalid format for --rename-comms-by-tids '{item}', " + "expected tid:name", file=sys.stderr) + sys.exit(1) + + if args.highlight_tasks: + for item in args.highlight_tasks.split(","): + parts = item.split(":") + if len(parts) == 1: + parts.append("red") + key, color = parts[0], parts[1] + args.highlight_tasks_map[key] = color + + analyzer = TaskAnalyzer(args) + analyzer.run() + +if __name__ == "__main__": + main() diff --git a/tools/perf/python/tracepoint.py b/tools/perf/python/tracepoint.py index 15b0c8268996..faa5311b8418 100755 --- a/tools/perf/python/tracepoint.py +++ b/tools/perf/python/tracepoint.py @@ -1,5 +1,6 @@ #! /usr/bin/env python # SPDX-License-Identifier: GPL-2.0 +"""Example showing how to enable a tracepoint and access its fields.""" # -*- python -*- # -*- coding: utf-8 -*- @@ -9,7 +10,7 @@ def change_proctitle(): try: import setproctitle setproctitle.setproctitle("tracepoint.py") - except: + except ImportError: print("Install the setproctitle python package to help with top and friends") def main(): @@ -29,7 +30,7 @@ def main(): evlist.open() evlist.mmap() - evlist.enable(); + evlist.enable() while True: evlist.poll(timeout = -1) diff --git a/tools/perf/python/treport.py b/tools/perf/python/treport.py new file mode 100755 index 000000000000..fb583786b17a --- /dev/null +++ b/tools/perf/python/treport.py @@ -0,0 +1,807 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: MIT +"""treport.py - perf report like tool written using textual.""" +from abc import ABC, abstractmethod +from typing import Callable, Dict, List, Optional, Set +import argparse +import os +import sys +import threading +import time +import perf +from rich.markup import escape +from rich.segment import Segment +from rich.style import Style +from textual import events, work +from textual.app import App, ComposeResult +from textual.binding import Binding +from textual.color import Color +from textual.scroll_view import ScrollView +from textual.strip import Strip +from textual.widgets import Footer, Header, TabbedContent, TabPane, Tree +from textual.widgets.tree import TreeNode + +def make_fixed_length_string(s: str, length: int, pad_char=' '): + """Make the string s a fixed length. + + Increases or decreases the length of s to be length. If the length is + increased then pad_char is inserted on the right. + """ + return s[:length] if len(s) > length else s.ljust(length, pad_char) + + +class FlameVisitor(ABC): + """Parent for visitor used by ProfileNode.flame_walk""" + @abstractmethod + def visit(self, node: Optional["ProfileNode"], width: int) -> None: + """Visit a profile node width the specified flame graph width. + + Args: + node: The `ProfileNode` for the current segment. This may be `None` + to represent a gap or an unknown portion of the stack. + width: The calculated width of the flame graph rectangle for this + node, which is proportional to its sample count. + """ + + +class ProfileNode: + """Represents a single node in a call stack tree. + + Generally a ProfileNode corresponds to a symbol in a call stack. + The root is special, its children are events and the events + children are process names. After the process name come the + samples. + + Attributes: + name (str): The name of the function, process or event. + value (int): The sample count for this node including counts from its + children. + parent (ProfileNode): The parent of this node, this node belongs to its + children. + children (Dict[str, ProfileNode]): A dictionary of child nodes, keyed by + their names. + """ + def __init__(self, name: str, parent: Optional["ProfileNode"]): + """Initializes a ProfileNode.""" + self.name = name + self.value: int = 0 + self.parent = parent if parent else self + self.children: Dict[str, ProfileNode] = {} + + def find_or_create_node(self, name: str) -> "ProfileNode": + """Finds a child node by name or creates it if it doesn't exist.""" + if name in self.children: + return self.children[name] + child = ProfileNode(name, self) + self.children[name] = child + return child + + def depth(self) -> int: + """The maximum depth of the call stack tree from this node down.""" + if not self.children: + return 1 + return max(child.depth() for child in self.children.values()) + 1 + + def process_event(self, sample, comm: str) -> None: + """Processes a single profiling event to update the call stack tree. + + Args: + sample: a single profiling sample. + comm: the command name of the sampled thread. + """ + period = sample.sample_period + self.value += period + + node = self.find_or_create_node(comm) + node.value += period + + if sample.callchain: + for entry in reversed(sample.callchain): + name = entry.symbol + if not name or name == "[unknown]": + name = entry.dso or "unknown" + if entry.ip: + name += f" 0x{entry.ip:x}" + node = node.find_or_create_node(name) + node.value += period + else: + name = sample.symbol + if not name or name == "[unknown]": + name = sample.dso or "unknown" + if sample.sample_ip: + name += f" 0x{sample.sample_ip:x}" + node = node.find_or_create_node(name) + node.value += period + + def sorted_children(self) -> List["ProfileNode"]: + """The children, largest value first. + + The root's children are events whose values aren't comparable, each is + 100% of itself, so they are kept in the order they were first seen + rather than reordering as they load. + """ + if self.parent is self: + return list(self.children.values()) + return sorted(self.children.values(), key=lambda pnode: pnode.value, reverse=True) + + def tree_label(self, root_value: int) -> str: + """Label for the node in a Tree, with a percentage of root_value. + + The percentage is highlighted with reversed colors. + """ + if root_value == 0: + return escape(self.name) + return f"{escape(self.name)} [r]{self.value / root_value * 100:.3g}%[/]" + + def largest_child(self) -> "ProfileNode": + """Finds the child with the highest value (sample count).""" + if self.children: + return self.sorted_children()[0] + return self + + def child_after(self, sought: "ProfileNode") -> "ProfileNode": + """Finds the next sibling after the given node, sorted by value.""" + found = False + for child in self.sorted_children(): + if child == sought: + found = True + elif found: + return child + return sought + + def child_before(self, sought: "ProfileNode") -> "ProfileNode": + """Finds the previous sibling before the given node, sorted by value.""" + last = None + for child in self.sorted_children(): + if child == sought: + return last if last else sought + last = child + return sought + + def has_parent(self, parent: "ProfileNode") -> bool: + """Checks if the parent node is an ancestor of this node.""" + p = self.parent + while True: + if p == parent: + return True + new_p = p.parent + if new_p == p: + break + p = new_p + return False + + def has_child(self, sought: "ProfileNode") -> bool: + """Checks if the sought node is a descendant of this node.""" + return sought.has_parent(self) + + def flame_walk(self, wanted_strip: int, cur_strip: int, parent_width: int, + selected: "ProfileNode", visitor: FlameVisitor) -> None: + """Recursively walks the tree to visit a single flame graph row. + + This method calculates the proportional width for each child + based on its value (sample count) relative to its parent. It + then invokes a `visitor` to process each segment of the flame + graph row. + + Args: + wanted_strip (int): The target depth (Y-axis) of the flame graph row + to generate. + cur_strip (int): The current depth of the traversal. + parent_width (int): The width of the parent of this node. + selected (ProfileNode): The currently selected node in the UI, used + to adjust rendering to highlight the + selected path. + visitor (FlameVisitor): A visitor object whose `visit` method is + called for each segment of the flame graph + row. + """ + if parent_width == 0: + return + + parent_selected = selected == self or self.has_parent(selected) + child_selected = not parent_selected and self.has_child(selected) + if not parent_selected and not child_selected: + # Branches of the tree with no node selected aren't drawn. + return + + # left_over is used to check for a gap after the children due + # to samples being in the parent. + left_over = parent_width + for child in self.sorted_children(): + if parent_selected: + if self.value: + desired_width = int((parent_width * child.value) / self.value) + else: + desired_width = parent_width // len(self.children) + if desired_width == 0: + # Nothing can be drawn for this node or later smaller children. + break + elif child == selected or child.has_child(selected): + desired_width = parent_width + else: + # A sibling or its child are selected, but not this branch. + continue + + # Either visit the wanted_strip or recurse to the next level. + if wanted_strip == cur_strip: + visitor.visit(child, desired_width) + else: + child.flame_walk(wanted_strip, cur_strip + 1, desired_width, + selected, visitor) + left_over -= desired_width + if left_over == 0: + # No space left to draw in. + break + + # Always visit the left_over regardless of the wanted_strip as there + # may be additional gap added to a line by a parent. + if left_over: + visitor.visit(None, left_over) + + def make_flame_strip(self, wanted_strip: int, parent_width: int, + cursor: "ProfileNode", selected: "ProfileNode", + theme_variables: Dict[str, str]) -> Strip: + """Creates a renderable 'Strip' for a single row of a flame graph. + + This method orchestrates the `flame_walk` traversal with a specialized + visitor to generate a list of segments. The segments are used by a`Strip` + object for rendering in the terminal. + + Args: + wanted_strip (int): The target depth (Y-axis) of the flame graph row. + parent_width (int): The total width (in characters) of the display + area. + cursor (ProfileNode): The node currently under the cursor, for + highlighting. + selected (ProfileNode): The node that is actively selected. + theme_variables(Dict): Values of colors for the textual theme. + + Returns: + Strip: A renderable strip of segments for the specified row. + """ + primary = Color.parse(theme_variables["primary"]) + secondary = Color.parse(theme_variables["secondary"]) + surface = Color.parse(theme_variables["surface"]) + def luminance(color: Color) -> float: + """Computes the luminance of a color from the rgb""" + return color.r * 0.299 + color.g * 0.587 + color.b * 0.114 + + # Set of styles for different flamegraph segments, the styles are + # cycled through to provide contrast. + normal_styles = [] + for x in range(0, 125, 25): + fgcolor = secondary.blend(primary, x/100) + if luminance(fgcolor) > luminance(surface): + bgcolor = surface.lighten(0.05+x/500) + else: + bgcolor = surface.darken(0.05+x/500) + normal_styles.append(Style(color=fgcolor.rich_color, + bgcolor=bgcolor.rich_color)) + + # Style for the selected flame graph node. + accent = Color.parse(theme_variables["accent"]) + accent_muted = Color.parse(theme_variables["accent-muted"]) + cursor_style = Style(color=accent.rich_color, bgcolor=accent_muted.rich_color) + + class StripVisitor(FlameVisitor): + """Visitor creating textual flame graph segments. + + Attributes: + segments (list): The textual segments that will be placed in a + `Strip`. + gap_width (int): The width of any outstanding gap between the + last and next node. + ctr (int): Used to adjust the flame graph segment's color. + """ + def __init__(self): + self.segments = [] + self.gap_width = 0 + self.ctr = wanted_strip + + def visit(self, node: Optional[ProfileNode], width: int) -> None: + if node: + if self.gap_width > 0: + self.segments.append(Segment( + make_fixed_length_string(" ", self.gap_width))) + self.gap_width = 0 + style = cursor_style + if node != cursor: + style = normal_styles[self.ctr % len(normal_styles)] + self.segments.append(Segment( + make_fixed_length_string(node.name, width), style)) + else: + self.gap_width += width + self.ctr += 1 + + visitor = StripVisitor() + self.flame_walk(wanted_strip, 0, parent_width, selected, visitor) + return Strip(visitor.segments) if visitor.segments else Strip.blank(parent_width) + + def find_node(self, sought_x: int, sought_y: int, parent_width: int, + selected: "ProfileNode") -> "ProfileNode": + """Finds the ProfileNode corresponding to specific X, Y coordinates. + + This translates a mouse click on a flame graph back to the + `ProfileNode` that it represents. + + Args: + sought_x (int): The X coordinate (character column). + sought_y (int): The Y coordinate (row or depth). + parent_width (int): The total width of the display area. + selected (ProfileNode): The currently selected node, which affects + layout. + + Returns: + Optional[ProfileNode]: The node found at the coordinates, or None. + + """ + class FindVisitor(FlameVisitor): + """Visitor locating a `ProfileNode`. + + Attributes: + x (int): offset within line. + found (Optional[ProfileNode]): located node + gap_width (int): The width of any outstanding gap between the + last and next node. + ctr (int): Used to adjust the flame graph segment's color. + """ + def __init__(self): + self.x = 0 + self.found = None + + def visit(self, node: Optional[ProfileNode], width: int) -> None: + if self.x <= sought_x < self.x + width: + self.found = node + self.x += width + + visitor = FindVisitor() + self.flame_walk(sought_y, 0, parent_width, selected, visitor) + return visitor.found + + +class FlameGraph(ScrollView): + """A scrollable widget to display a flame graph from a profile. + + Attributes: + root (ProfileNode): Root of the profile tree. + cursor (ProfileNode): Currently highlighted cursor node. + selected (ProfileNode): The currently selected node for zooming. + """ + + # Define key bindings for navigating the flame graph. + # Allows movement with vim-style keys (h,j,k,l) and arrow keys. + BINDINGS = [ + Binding("j,down", "move_down", "Down", key_display="↓", + tooltip="Move cursor down to largest child"), + Binding("k,up", "move_up", "Up", key_display="↑", + tooltip="Move cursor up to parent"), + Binding("l,right", "move_right", "Right", key_display="→", + tooltip="Move cursor to the right sibling"), + Binding("h,left", "move_left", "Left", key_display="←", + tooltip="Move cursor to the left sibling"), + Binding("enter", "zoom_in", "Zoom In", + tooltip="Expand the cursor's node to be screen width"), + Binding("escape", "zoom_out", "Zoom Out", + tooltip="Zoom out to initial view."), + ] + + # Default CSS for the widget to ensure it fills its container's width. + DEFAULT_CSS = """ + FlameGraph { + width: 100%; + } + """ + + def __init__(self, root: ProfileNode, lock: threading.Lock, *pos_args, **kwargs): + """Initialize the FlameGraph widget. + + The lock must be held when reading the profile as it may be being built + in another thread. + """ + super().__init__(*pos_args, **kwargs) + self.root = root + self.profile_lock = lock + self.cursor = root + self.selected = root + + def action_move_down(self) -> None: + """Handle key press down.""" + with self.profile_lock: + self.cursor = self.cursor.largest_child() + self.refresh() + + def action_move_up(self) -> None: + """Handle key press up.""" + if self.cursor.parent != self.cursor.parent.parent: + self.cursor = self.cursor.parent + self.refresh() + + def action_move_right(self) -> None: + """Handle key press right.""" + with self.profile_lock: + self.cursor = self.cursor.parent.child_after(self.cursor) + self.refresh() + + def action_move_left(self) -> None: + """Handle key press left.""" + with self.profile_lock: + self.cursor = self.cursor.parent.child_before(self.cursor) + self.refresh() + + def action_zoom_in(self) -> None: + """Handle key press zoom in.""" + self.selected = self.cursor + self.refresh() + + def action_zoom_out(self) -> None: + """Handle key press zoom out.""" + self.selected = self.root + self.refresh() + + def render_line(self, y: int) -> Strip: + """Render a single line (row) of the flame graph.""" + _, scroll_y = self.scroll_offset + y += scroll_y + with self.profile_lock: + return self.root.make_flame_strip(y, self.size.width, self.cursor, + self.selected, self.app.theme_variables) + + def profile_changed(self) -> None: + """Resize and redraw after the profile changed.""" + with self.profile_lock: + self.styles.height = self.root.depth() + self.refresh() + + def on_mount(self) -> None: + """Set the height of the widget when it is displayed.""" + self.profile_changed() + + def on_click(self, click: events.Click) -> None: + """Handles a mouse click and update the cursor position.""" + _, scroll_y = self.scroll_offset + y = scroll_y + click.y + with self.profile_lock: + clicked_node = self.root.find_node(click.x, y, self.size.width, + self.selected) + if clicked_node: + self.cursor = clicked_node + self.refresh() + + +class ProfileTree(Tree): + """A tree view of the profile that can be updated while it is built. + + Tree nodes are created lazily when expanded, the data of each tree node is + its ProfileNode. The lock must be held when reading the profile. + """ + + def __init__(self, root: ProfileNode, lock: threading.Lock, *pos_args, **kwargs): + super().__init__("Profile", *pos_args, **kwargs) + self.profile = root + self.profile_lock = lock + # Events whose initial expansion has been done. + self.seen_events: Set[str] = set() + + def pnode_of(self, tnode: TreeNode) -> ProfileNode: + """The ProfileNode shown by tnode, the tree's root shows the profile's root.""" + return tnode.data if tnode.data is not None else self.profile + + def event_value(self, pnode: ProfileNode) -> int: + """Value of the event containing pnode, percentages are relative to it.""" + while pnode.parent is not self.profile and pnode.parent is not pnode: + pnode = pnode.parent + return pnode.value + + @staticmethod + def expanded_descendants(tnode: TreeNode, expanded: Set[ProfileNode]) -> None: + """Add the profile nodes of the expanded descendants of tnode to expanded.""" + for tchild in tnode.children: + if tchild.is_expanded and tchild.data is not None: + expanded.add(tchild.data) + if tchild.children: + ProfileTree.expanded_descendants(tchild, expanded) + + def sync_node(self, tnode: TreeNode, expanded: Optional[Set[ProfileNode]] = None) -> None: + """Update the children of tnode, and expanded descendants, from the profile. + + expanded holds the profile nodes of descendants that were expanded + before an ancestor's children were recreated. The lock must be held. + """ + children = self.pnode_of(tnode).sorted_children() + if expanded is not None or [tchild.data for tchild in tnode.children] != children: + # Recreate the children in the new order keeping the expanded + # ones, and their expanded descendants. + if expanded is None: + expanded = set() + self.expanded_descendants(tnode, expanded) + tnode.remove_children() + ancestors: Set[ProfileNode] = set() + for pnode in expanded: + parent = pnode.parent + while parent is not None and parent not in ancestors: + ancestors.add(parent) + parent = parent.parent + for child in children: + label = child.tree_label(self.event_value(child)) + new = tnode.add(label, child, allow_expand=bool(child.children)) + if child in expanded: + new.expand() + if child in expanded or child in ancestors: + self.sync_node(new, expanded) + return + for tchild in tnode.children: + child = self.pnode_of(tchild) + tchild.set_label(child.tree_label(self.event_value(child))) + tchild.allow_expand = bool(child.children) + if tchild.is_expanded or tchild.children: + self.sync_node(tchild) + + def expand_largest(self, tnode: TreeNode) -> None: + """Expand the chain of largest children below tnode. + + The lock must be held. + """ + while self.pnode_of(tnode).children: + self.sync_node(tnode) + tnode = tnode.children[0] + tnode.expand() + + def find(self, tnode: TreeNode, pnode: ProfileNode) -> Optional[TreeNode]: + """Find the visible tree node for pnode.""" + for tchild in tnode.children: + if tchild.data is pnode: + return tchild + if tchild.is_expanded: + found = self.find(tchild, pnode) + if found: + return found + return None + + def sync(self) -> None: + """Update the tree from the profile keeping the expanded nodes and cursor.""" + cursor = self.cursor_node.data if self.cursor_node else None + with self.profile_lock: + self.root.expand() + self.sync_node(self.root) + for tnode in self.root.children: + name = self.pnode_of(tnode).name + if name in self.seen_events: + continue + self.seen_events.add(name) + self.expand_largest(tnode) + # If there is only one event, expand it also. + if len(self.root.children) == 1: + tnode.expand() + if cursor is not None: + found = self.find(self.root, cursor) + if found: + self.move_cursor(found) + + def on_tree_node_expanded(self, event: Tree.NodeExpanded) -> None: + """Create the children of a node when it is expanded.""" + if event.node is not self.root: + with self.profile_lock: + self.sync_node(event.node) + + +class ReportApp(App): + """A Textual application to display profiling data.""" + + # The ^q binding is implied but having it here adds it in the Footer. + BINDINGS = [ + Binding(key="^q", action="quit", description="Quit", + tooltip="Quit the app"), + ] + + def __init__(self, input_file: str, root: Optional[ProfileNode] = None): + """Initialize the application. + + If root isn't given the profile is loaded from input_file in the + background after the application starts, the views show the profile + as it is built. + """ + super().__init__() + self.input_file = input_file + self.root = root if root else ProfileNode("root", parent=None) + self.loaded = root is not None + # Held when reading or changing the profile, which is built in a + # background thread. + self.profile_lock = threading.Lock() + # The profile being built in a background thread. + self.loading: Optional[ProfileBuilder] = None + + def compose(self) -> ComposeResult: + """Composes the user interface of the application.""" + yield Header() + with TabbedContent(initial="report"): + with TabPane("Report", id="report"): + yield ProfileTree(self.root, self.profile_lock) + with TabPane("Flame Graph", id="flame"): + yield FlameGraph(self.root, self.profile_lock) + yield Footer() + + def on_mount(self) -> None: + """Start loading the profile unless it was given.""" + self.sub_title = self.input_file + if self.loaded: + self.update_views() + else: + self.sub_title = f"Loading {self.input_file}" + self.loading = ProfileBuilder(self.root, self.profile_lock) + self.load_profile() + + def update_views(self) -> None: + """Show the latest state of the profile.""" + self.query_one(ProfileTree).sync() + self.query_one(FlameGraph).profile_changed() + + @work(thread=True, exclusive=True) + def load_profile(self) -> None: + """Build the profile in a thread so the UI stays responsive.""" + loading = self.loading + assert loading is not None + profile = loading + + def progress() -> None: + try: + self.call_from_thread(self.update_progress, profile) + except RuntimeError: + # The app is no longer running. + profile.cancelled = True + + profile.progress = progress + try: + profile.read(self.input_file) + except LoadCancelled: + return + except (OSError, ValueError, RuntimeError) as e: + self.call_from_thread(self.exit, None, 1, + f"Error processing {self.input_file}: {e}") + return + finally: + profile.progress = None + self.loading = None + if not profile.cancelled: + self.call_from_thread(self.profile_loaded, profile) + + def update_progress(self, profile: "ProfileBuilder") -> None: + """Show how much of the file has been processed and the profile so far.""" + self.sub_title = (f"Loading {self.input_file}: {profile.nr_samples:,} samples" + f"{profile.duration_str()}") + self.update_views() + + def profile_loaded(self, profile: "ProfileBuilder") -> None: + """Called on the UI thread when loading completes.""" + self.loaded = True + self.sub_title = (f"{self.input_file}: {profile.nr_samples:,} samples" + f"{profile.duration_str()}") + self.update_views() + + def cancel_loading(self) -> None: + """Stop a background load, noticed at the next progress interval.""" + loading = self.loading + if loading is not None: + loading.cancelled = True + + async def action_quit(self) -> None: + """Quit, stopping any background load.""" + self.cancel_loading() + await super().action_quit() + + def on_unmount(self) -> None: + """Stop any background load when the app exits.""" + self.cancel_loading() + + +class LoadCancelled(Exception): + """Raised from the sample callback to stop processing events early.""" + + +class ProfileBuilder: + """Constructs a profile tree from a stream of events.""" + # Number of samples between checks for cancellation and progress. + PROGRESS_INTERVAL = 1000 + # Minimum and maximum time between calls to the progress callback. + PROGRESS_SECONDS = 1.0 + PROGRESS_MAX_SECONDS = 10.0 + # Updating the views costs more as the profile grows, so the time between + # progress calls grows as this fraction of the time spent loading. This + # bounds the fraction of the load time spent updating the views. + PROGRESS_FRACTION = 0.25 + + def __init__(self, root: Optional[ProfileNode] = None, + lock: Optional[threading.Lock] = None) -> None: + """Build into root, holding lock when changing it.""" + self.root = root if root else ProfileNode("root", parent=None) + self.profile_lock = lock if lock else threading.Lock() + self.session: Optional[perf.session] = None + self.nr_samples = 0 + self.first_time = 0 + self.last_time = 0 + # Set, possibly from another thread, to stop processing events. + self.cancelled = False + # Called periodically, see PROGRESS_FRACTION, while processing events. + self.progress: Optional[Callable[[], None]] = None + self.start_progress = time.monotonic() + self.last_progress = self.start_progress + + def duration_str(self) -> str: + """Description of the time covered by the processed samples.""" + if not self.last_time: + return "" + return f", {(self.last_time - self.first_time) / 1e9:.3f}s of trace" + + def comm(self, sample) -> str: + """The command name of the thread of the sample.""" + pid = sample.sample_pid + try: + assert self.session + thread = self.session.find_thread(pid, sample.sample_tid) + return (thread.comm() if thread else None) or f"unknown ({pid})" + except (OSError, ValueError, KeyError, RuntimeError, TypeError, AttributeError, + AssertionError): + return f"unknown ({pid})" + + def process_event(self, sample) -> None: + """Called by session.process_events to update the profile tree.""" + self.nr_samples += 1 + if self.nr_samples % self.PROGRESS_INTERVAL == 0: + if self.cancelled: + raise LoadCancelled() + now = time.monotonic() + interval = min(max(self.PROGRESS_SECONDS, + (now - self.start_progress) * self.PROGRESS_FRACTION), + self.PROGRESS_MAX_SECONDS) + if self.progress and now - self.last_progress >= interval: + # Must not hold the lock as the callback may read the profile. + self.progress() + # Time from when the callback, that may block, returns. + self.last_progress = time.monotonic() + sample_time = sample.sample_time + if sample_time: + if not self.first_time or sample_time < self.first_time: + self.first_time = sample_time + self.last_time = max(self.last_time, sample_time) + ev_name = str(sample.evsel)[6:-1] + comm = self.comm(sample) + with self.profile_lock: + ev_root = self.root.find_or_create_node(ev_name) + ev_root.process_event(sample, comm) + + def read(self, input_file: str) -> None: + """Process the events in input_file, raising on errors.""" + if self.cancelled: + raise LoadCancelled() + self.start_progress = self.last_progress = time.monotonic() + try: + self.session = perf.session(perf.data(input_file), sample=self.process_event) + self.session.process_events() + finally: + # Break the reference cycle between the session and the callback. + self.session = None + + +def main() -> None: + """Parse arguments and run the app.""" + parser = argparse.ArgumentParser( + description="TUI report and flame graph using perf python module.") + parser.add_argument("-i", "--input", help="input perf.data file") + args = parser.parse_args() + + input_file = args.input or "perf.data" + if input_file == "-": + # The interactive UI reads the keyboard from stdin. + print("Error: reading perf.data from stdin isn't supported.", file=sys.stderr) + sys.exit(1) + if not os.path.exists(input_file): + print(f"Error: {input_file} not found. (try 'perf record' first)", file=sys.stderr) + sys.exit(1) + + # The app starts immediately and builds the profile in the background. + app = ReportApp(input_file) + app.run() + sys.exit(app.return_code or 0) + + +if __name__ == "__main__": + main() diff --git a/tools/perf/python/ttimechart.py b/tools/perf/python/ttimechart.py new file mode 100755 index 000000000000..d7ba0276a9fb --- /dev/null +++ b/tools/perf/python/ttimechart.py @@ -0,0 +1,1806 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0 +"""ttimechart.py - interactive perf timechart written using textual. + +Reads a perf.data file, typically created with 'perf timechart record', and +displays per-CPU and per-task timelines in the terminal. Scheduler +(sched:sched_switch, sched:sched_wakeup), power (power:cpu_idle, +power:cpu_frequency) and I/O syscall (perf timechart record -I) tracepoints +are understood. Unlike 'perf timechart', which writes an SVG file, the +timeline can be zoomed, panned and queried interactively. + +Usage: + perf timechart record -- <workload> + perf timechart --tui +or: + perf script ttimechart [-i perf.data] +""" +from __future__ import annotations + +from abc import ABC, abstractmethod +import argparse +import bisect +from collections import defaultdict +from dataclasses import dataclass, replace +import math +import os +import sys +import threading +from time import monotonic +from typing import Any, Callable, Dict, List, Mapping, Optional, Sequence, Tuple + +import perf +from rich.segment import Segment +from rich.style import Style +from textual import events, on, work +from textual.app import App, ComposeResult +from textual.binding import Binding +from textual.color import Color +from textual.geometry import Size +from textual.message import Message +from textual.reactive import reactive +from textual.scroll_view import ScrollView +from textual.strip import Strip +from textual.widgets import DataTable, Footer, Header, Static, TabbedContent, TabPane +from textual.widgets.data_table import CellDoesNotExist, RowDoesNotExist + +# Task states, SLEEPING and UNKNOWN aren't drawn. +STATE_UNKNOWN = -1 +STATE_SLEEPING = 0 +STATE_RUNNING = 1 +STATE_WAITING = 2 +STATE_BLOCKED = 3 +NUM_STATES = 4 +STATE_NAMES = { + STATE_UNKNOWN: "unknown", + STATE_SLEEPING: "sleeping", + STATE_RUNNING: "running", + STATE_WAITING: "runnable (waiting for a CPU)", + STATE_BLOCKED: "blocked (uninterruptible)", +} + +# I/O types, matching builtin-timechart.c. +IOTYPE_READ = 0 +IOTYPE_WRITE = 1 +IOTYPE_SYNC = 2 +IOTYPE_TX = 3 +IOTYPE_RX = 4 +IOTYPE_POLL = 5 +NUM_IOTYPES = 6 +IOTYPE_NAMES = ["read", "write", "sync", "tx", "rx", "poll"] + +IO_SYSCALLS = { + "read": IOTYPE_READ, "pread64": IOTYPE_READ, "readv": IOTYPE_READ, + "preadv": IOTYPE_READ, + "write": IOTYPE_WRITE, "pwrite64": IOTYPE_WRITE, "writev": IOTYPE_WRITE, + "pwritev": IOTYPE_WRITE, + "sync": IOTYPE_SYNC, "sync_file_range": IOTYPE_SYNC, "fsync": IOTYPE_SYNC, + "msync": IOTYPE_SYNC, + "recvfrom": IOTYPE_RX, "recvmmsg": IOTYPE_RX, "recvmsg": IOTYPE_RX, + "sendto": IOTYPE_TX, "sendmsg": IOTYPE_TX, "sendmmsg": IOTYPE_TX, + "epoll_pwait": IOTYPE_POLL, "epoll_wait": IOTYPE_POLL, "poll": IOTYPE_POLL, + "ppoll": IOTYPE_POLL, "pselect6": IOTYPE_POLL, "select": IOTYPE_POLL, +} + +# Trace flags for interrupt context in the common_flags tracepoint field. +TRACE_FLAG_HARDIRQ = 0x08 +TRACE_FLAG_SOFTIRQ = 0x10 +# Value of state in power:cpu_idle when leaving idle. +PWR_EVENT_EXIT = 0xffffffff +# Width of the row label column. +LABEL_WIDTH = 28 +# Characters for drawing fractional bars. +BARS = " ▁▂▃▄▅▆▇█" +NSEC_PER_SEC = 1_000_000_000 + + +def fmt_duration(nsecs: float) -> str: + """Format a duration in nanoseconds with an appropriate unit.""" + if nsecs >= NSEC_PER_SEC: + return f"{nsecs / NSEC_PER_SEC:.3f}s" + if nsecs >= 1_000_000: + return f"{nsecs / 1_000_000:.3f}ms" + if nsecs >= 1_000: + return f"{nsecs / 1_000:.3f}us" + return f"{nsecs:.0f}ns" + + +def escape(text: str) -> str: + """Escape text, such as a task name, for use in textual markup. + + rich.markup.escape doesn't escape tags like "[1]", the name given to tasks + with an unknown command, but textual fails to parse them. + """ + return text.replace("[", "\\[") + + +def make_fixed_length_string(s: str, length: int, pad_char: str = ' ') -> str: + """Truncate or right pad s so that it is length characters long.""" + return s[:length] if len(s) > length else s.ljust(length, pad_char) + + +def bar_char(frac: float) -> str: + """A block character whose height represents frac, in the range [0, 1].""" + if frac <= 0: + return BARS[0] + return BARS[max(1, min(8, round(frac * 8)))] + + +class SegmentList: + """A time ordered list of non-overlapping [start, end) segments. + + Each segment has a small integer key, used for computing coverage, and + arbitrary associated data. + """ + def __init__(self) -> None: + self.starts: List[int] = [] + self.ends: List[int] = [] + self.keys: List[int] = [] + self.data: List[Any] = [] + + def __len__(self) -> int: + return len(self.starts) + + def add(self, start: int, end: int, key: int, data: Any = None) -> None: + """Append a segment, segments must be added in time order.""" + if end <= start: + return + if self.ends and start < self.ends[-1]: + # Clip overlaps caused by inconsistent data. + start = self.ends[-1] + if end <= start: + return + self.starts.append(start) + self.ends.append(end) + self.keys.append(key) + self.data.append(data) + + def find(self, time: float) -> int: + """Index of the segment containing time or -1.""" + i = bisect.bisect_right(self.starts, time) - 1 + if i >= 0 and time < self.ends[i]: + return i + return -1 + + def last_before(self, time: float) -> int: + """Index of the last segment starting at or before time or -1.""" + return bisect.bisect_right(self.starts, time) - 1 + + def next_change(self, time: float) -> Optional[int]: + """The first segment start or end after time.""" + candidates = [] + i = bisect.bisect_right(self.starts, time) + if i < len(self.starts): + candidates.append(self.starts[i]) + i = bisect.bisect_right(self.ends, time) + if i < len(self.ends): + candidates.append(self.ends[i]) + return min(candidates) if candidates else None + + def prev_change(self, time: float) -> Optional[int]: + """The last segment start or end before time.""" + candidates = [] + i = bisect.bisect_left(self.starts, time) - 1 + if i >= 0: + candidates.append(self.starts[i]) + i = bisect.bisect_left(self.ends, time) - 1 + if i >= 0: + candidates.append(self.ends[i]) + return max(candidates) if candidates else None + + def coverage(self, t0: float, dt: float, width: int, nkeys: int, + weight_fn=None) -> List[List[float]]: + """Time covered by each key in each of width columns of size dt. + + If weight_fn is given then, rather than the time covered, the time + covered multiplied by weight_fn(data) is accumulated. + """ + cols = [[0.0] * nkeys for _ in range(width)] + t1 = t0 + dt * width + i = bisect.bisect_right(self.ends, t0) + num = len(self.starts) + while i < num and self.starts[i] < t1: + start = max(self.starts[i], t0) + end = min(self.ends[i], t1) + key = self.keys[i] + weight = weight_fn(self.data[i]) if weight_fn else 1.0 + x0 = min(int((start - t0) / dt), width - 1) + x1 = min(int((end - t0) / dt), width - 1) + if x0 == x1: + cols[x0][key] += (end - start) * weight + else: + cols[x0][key] += (t0 + (x0 + 1) * dt - start) * weight + for x in range(x0 + 1, x1): + cols[x][key] += dt * weight + cols[x1][key] += (end - (t0 + x1 * dt)) * weight + i += 1 + return cols + + +class Task: + """Scheduling and I/O history of a single thread.""" + def __init__(self, tid: int, comm: str) -> None: + self.tid = tid + self.comm = comm + self.comms: List[str] = [comm] + self.state = STATE_UNKNOWN + self.since = 0 + self.cpu = -1 + # Segments with a state key and the CPU as data. + self.segs = SegmentList() + # Segments with an I/O type key and (fd, ret) as data. + self.io = SegmentList() + self.io_pending: Optional[Tuple[int, int, int]] = None + # Wakeups of this task as (time, waker tid) pairs. + self.wakeups: List[Tuple[int, int]] = [] + self.totals = [0] * NUM_STATES + self.switches = 0 + self.io_bytes = 0 + + def name(self) -> str: + """Name for the task used in labels.""" + return f"{self.comm} ({self.tid})" + + def set_comm(self, comm: Optional[str]) -> None: + """Update the task's command name.""" + if not comm or comm == self.comm: + return + self.comm = comm + if comm not in self.comms: + self.comms.append(comm) + + def change_state(self, time: int, state: int, cpu: int = -1) -> None: + """Record the current state as a segment and switch to a new state.""" + if self.state in (STATE_RUNNING, STATE_WAITING, STATE_BLOCKED) and time > self.since: + self.segs.add(self.since, time, self.state, self.cpu) + self.totals[self.state] += time - self.since + self.state = state + self.since = time + if cpu >= 0: + self.cpu = cpu + + def passes_filter(self, filters: Sequence[str]) -> bool: + """Does the task match one of the process filters (pid or name)?""" + if not filters: + return True + return any(f == str(self.tid) or f in self.comms for f in filters) + + +class Cpu: + """Activity on a single CPU.""" + def __init__(self, cpu: int) -> None: + self.cpu = cpu + self.cur_tid = -1 + self.since = 0 + # Busy segments, key 1, with the running tid as data. + self.run = SegmentList() + # Idle state segments, key 1, with the C-state as data. + self.cstate = SegmentList() + self.cstate_cur: Optional[Tuple[int, int]] = None + # Frequency segments, key 1, with the frequency in kHz as data. + self.pstate = SegmentList() + self.pstate_cur: Optional[Tuple[int, int]] = None + + +class LoadCancelled(Exception): + """Raised from the sample callback to stop processing events early.""" + + +class TimechartData: + """Builds per-task and per-CPU timelines from perf events. + + The data may be displayed while it is loaded in another thread, lock must + be held when modifying it or when reading it from another thread. + """ + # Number of samples between checks for cancellation and progress. + PROGRESS_INTERVAL = 1000 + # Minimum and maximum time between calls to the progress callback. + PROGRESS_SECONDS = 1.0 + PROGRESS_MAX_SECONDS = 10.0 + # Updating the views costs more as more data is loaded, so the time between + # progress calls grows as this fraction of the time spent loading. This + # bounds the fraction of the load time spent updating the views. + PROGRESS_FRACTION = 0.25 + + def __init__(self) -> None: + self.tasks: Dict[int, Task] = {} + self.cpus: Dict[int, Cpu] = {} + self.first_time = 0 + self.last_time = 0 + self.min_freq = 0 + self.max_freq = 0 + self.max_cstate = 0 + self.sched_events = 0 + self.power_events = 0 + self.io_events = 0 + self.nr_samples = 0 + self.unhandled: Dict[str, int] = defaultdict(int) + self.session: Optional[perf.session] = None + # Evsel name and event handler, keyed by sample ID. + self._handlers: Dict[int, Tuple[str, Optional[Callable[[int, perf.sample_event], + None]]]] = {} + self.lock = threading.Lock() + # Set, possibly from another thread, to stop processing events. + self.cancelled = False + # Called periodically, see PROGRESS_FRACTION, while processing events. + self.progress: Optional[Callable[[], None]] = None + self.start_progress = monotonic() + self.last_progress = self.start_progress + + def has_events(self) -> bool: + """Were any events that can be displayed processed?""" + return bool(self.sched_events or self.power_events or self.io_events) + + def task(self, tid: int, comm: Optional[str] = None) -> Task: + """Find or create a task.""" + task = self.tasks.get(tid) + if task is None: + if not comm and self.session: + try: + thread = self.session.find_thread(tid, tid) + comm = thread.comm() if thread else None + except (OSError, ValueError, KeyError, RuntimeError, TypeError, AttributeError): + comm = None + task = Task(tid, comm or f"[{tid}]") + self.tasks[tid] = task + else: + task.set_comm(comm) + return task + + def cpu(self, cpu: int) -> Cpu: + """Find or create a CPU.""" + c = self.cpus.get(cpu) + if c is None: + c = Cpu(cpu) + self.cpus[cpu] = c + return c + + def sched_switch(self, time: int, cpu: int, prev_tid: int, prev_comm: Optional[str], + prev_state: int, next_tid: int, next_comm: Optional[str]) -> None: + """Process a sched:sched_switch event.""" + self.sched_events += 1 + c = self.cpu(cpu) + if c.cur_tid == -1 and prev_tid != 0: + # Assume the task was running from the start of the trace. + c.cur_tid = prev_tid + c.since = self.first_time + if c.cur_tid > 0: + c.run.add(c.since, time, 1, c.cur_tid) + c.cur_tid = next_tid + c.since = time + + if prev_tid != 0: + prev = self.task(prev_tid, prev_comm) + if prev.state == STATE_UNKNOWN: + prev.state = STATE_RUNNING + prev.since = self.first_time + prev.cpu = cpu + # Ignore bits like TASK_REPORT_MAX used to report preemption. + state = prev_state & 0xff + if state == 0: + new_state = STATE_WAITING + elif state & 2: + new_state = STATE_BLOCKED + else: + new_state = STATE_SLEEPING + prev.change_state(time, new_state) + prev.switches += 1 + + if next_tid != 0: + nxt = self.task(next_tid, next_comm) + nxt.change_state(time, STATE_RUNNING, cpu) + + def sched_wakeup(self, time: int, wakee: int, comm: Optional[str], waker: int) -> None: + """Process a sched:sched_wakeup or sched:sched_wakeup_new event.""" + self.sched_events += 1 + if wakee == 0: + return + task = self.task(wakee, comm) + task.wakeups.append((time, waker)) + if task.state in (STATE_UNKNOWN, STATE_SLEEPING, STATE_BLOCKED): + task.change_state(time, STATE_WAITING) + + def cstate_start(self, time: int, cpu: int, state: int) -> None: + """Enter an idle state.""" + self.power_events += 1 + c = self.cpu(cpu) + if c.cstate_cur: + c.cstate.add(c.cstate_cur[0], time, 1, c.cstate_cur[1]) + c.cstate_cur = (time, state) + self.max_cstate = max(self.max_cstate, state) + + def cstate_end(self, time: int, cpu: int) -> None: + """Leave an idle state.""" + self.power_events += 1 + c = self.cpu(cpu) + if c.cstate_cur: + c.cstate.add(c.cstate_cur[0], time, 1, c.cstate_cur[1]) + c.cstate_cur = None + + def pstate_change(self, time: int, cpu: int, freq: int) -> None: + """Change of CPU frequency, freq is in kHz.""" + if freq <= 0 or freq > 8000000: + return + self.power_events += 1 + c = self.cpu(cpu) + if c.pstate_cur: + c.pstate.add(c.pstate_cur[0], time, 1, c.pstate_cur[1]) + c.pstate_cur = (time, freq) + self.max_freq = max(self.max_freq, freq) + self.min_freq = freq if not self.min_freq else min(self.min_freq, freq) + + def io_enter(self, time: int, tid: int, iotype: int, fd: int) -> None: + """Entry to an I/O syscall.""" + self.io_events += 1 + self.task(tid).io_pending = (time, iotype, fd) + + def io_exit(self, time: int, tid: int, iotype: int, ret: int) -> None: + """Exit from an I/O syscall.""" + self.io_events += 1 + task = self.task(tid) + pending = task.io_pending + task.io_pending = None + if not pending or pending[1] != iotype: + return + start = pending[0] + task.io.add(start, max(time, start + 1), iotype, (pending[2], ret)) + if ret > 0 and iotype in (IOTYPE_READ, IOTYPE_WRITE, IOTYPE_TX, IOTYPE_RX): + task.io_bytes += ret + + def _on_sched_switch(self, time: int, sample: perf.sample_event) -> None: + self.sched_switch(time, sample.sample_cpu, sample.prev_pid, + getattr(sample, "prev_comm", None), sample.prev_state, + sample.next_pid, getattr(sample, "next_comm", None)) + + def _on_sched_wakeup(self, time: int, sample: perf.sample_event) -> None: + waker = getattr(sample, "common_pid", sample.sample_tid) + flags = getattr(sample, "common_flags", 0) + if flags & (TRACE_FLAG_HARDIRQ | TRACE_FLAG_SOFTIRQ): + waker = -1 + self.sched_wakeup(time, sample.pid, getattr(sample, "comm", None), waker) + + def _cstate(self, time: int, cpu: int, state: int) -> None: + if (state & 0xffffffff) == PWR_EVENT_EXIT: + self.cstate_end(time, cpu) + else: + self.cstate_start(time, cpu, state) + + def _on_cpu_idle(self, time: int, sample: perf.sample_event) -> None: + self._cstate(time, sample.cpu_id, sample.state) + + def _on_power_start(self, time: int, sample: perf.sample_event) -> None: + self._cstate(time, sample.cpu_id, sample.value) + + def _on_power_end(self, time: int, sample: perf.sample_event) -> None: + self.cstate_end(time, sample.sample_cpu) + + def _on_cpu_frequency(self, time: int, sample: perf.sample_event) -> None: + self.pstate_change(time, sample.cpu_id, sample.state) + + def _on_power_frequency(self, time: int, sample: perf.sample_event) -> None: + self.pstate_change(time, sample.cpu_id, sample.value) + + def _handler_for(self, name: str) -> Optional[Callable[[int, perf.sample_event], None]]: + """Find the handler for events with the given evsel name.""" + handlers: Dict[str, Callable[[int, perf.sample_event], None]] = { + "sched:sched_switch": self._on_sched_switch, + "sched:sched_wakeup": self._on_sched_wakeup, + "sched:sched_wakeup_new": self._on_sched_wakeup, + "power:cpu_idle": self._on_cpu_idle, + "power:power_start": self._on_power_start, + "power:power_end": self._on_power_end, + "power:cpu_frequency": self._on_cpu_frequency, + "power:power_frequency": self._on_power_frequency, + } + if name in handlers: + return handlers[name] + if name.startswith("syscalls:sys_enter_") and name[19:] in IO_SYSCALLS: + iotype = IO_SYSCALLS[name[19:]] + return lambda time, sample: self.io_enter(time, sample.sample_tid, iotype, + getattr(sample, "fd", -1)) + if name.startswith("syscalls:sys_exit_") and name[18:] in IO_SYSCALLS: + iotype = IO_SYSCALLS[name[18:]] + return lambda time, sample: self.io_exit(time, sample.sample_tid, iotype, + sample.ret) + return None + + def process_event(self, sample: perf.sample_event) -> None: + """Callback from perf.session for each sample.""" + self.nr_samples += 1 + if self.nr_samples % self.PROGRESS_INTERVAL == 0: + if self.cancelled: + raise LoadCancelled() + now = monotonic() + interval = min(max(self.PROGRESS_SECONDS, + (now - self.start_progress) * self.PROGRESS_FRACTION), + self.PROGRESS_MAX_SECONDS) + if self.progress and now - self.last_progress >= interval: + with self.lock: + self.update_comms() + # Must not hold the lock as the callback may read the data. + self.progress() + # Time from when the callback, that may block, returns. + self.last_progress = monotonic() + with self.lock: + self._process_event(sample) + + def _process_event(self, sample: perf.sample_event) -> None: + """Update the data from a sample, the lock must be held.""" + time = sample.sample_time + if not self.first_time or time < self.first_time: + self.first_time = time + self.last_time = max(self.last_time, time) + + # Computing the evsel name and matching it is relatively expensive, + # so cache the result by sample ID. Each ID belongs to a single evsel. + sample_id = sample.sample_id + cached = self._handlers.get(sample_id) + if cached is None: + name = str(sample.evsel) + if name.startswith("evsel(") and name.endswith(")"): + name = name[6:-1] + cached = (name, self._handler_for(name)) + self._handlers[sample_id] = cached + name, handler = cached + if handler is None: + self.unhandled[name] += 1 + return + try: + handler(time, sample) + except AttributeError: + self.unhandled[name] += 1 + + def update_comms(self) -> None: + """Refresh task command names from the session, the lock must be held.""" + if not self.session: + return + for tid, task in self.tasks.items(): + try: + thread = self.session.find_thread(tid, tid) + if thread: + task.set_comm(thread.comm()) + except (OSError, ValueError, KeyError, RuntimeError, TypeError, AttributeError): + pass + + def finish(self) -> None: + """Close open segments at the end of the trace.""" + with self.lock: + self.update_comms() + end = self.last_time + for task in self.tasks.values(): + task.change_state(end, STATE_UNKNOWN) + for c in self.cpus.values(): + if c.cur_tid > 0: + c.run.add(c.since, end, 1, c.cur_tid) + if c.cstate_cur: + c.cstate.add(c.cstate_cur[0], end, 1, c.cstate_cur[1]) + c.cstate_cur = None + if c.pstate_cur: + c.pstate.add(c.pstate_cur[0], end, 1, c.pstate_cur[1]) + c.pstate_cur = None + + def fmt_time(self, time: float) -> str: + """Format an absolute timestamp relative to the trace start.""" + return f"{(time - self.first_time) / NSEC_PER_SEC:.6f}s" + + def task_name(self, tid: int) -> str: + """Name of a task, or a description for special tids.""" + if tid == 0: + return "idle" + if tid < 0: + return "interrupt" + task = self.tasks.get(tid) + return task.name() if task else f"[{tid}]" + + def sched_tasks(self, filters: Sequence[str]) -> List[Task]: + """Tasks with scheduling history passing the filters, sorted by tid.""" + return sorted((t for t in self.tasks.values() + if len(t.segs) and t.passes_filter(filters)), + key=lambda t: t.tid) + + def io_tasks(self, filters: Sequence[str]) -> List[Task]: + """Tasks with I/O passing the filters, most I/O first.""" + return sorted((t for t in self.tasks.values() + if len(t.io) and t.passes_filter(filters)), + key=lambda t: -len(t.io)) + + def dump(self, power_only: bool, tasks_only: bool, filters: Sequence[str]) -> None: + """Print a plain text summary, for use without a terminal UI.""" + duration = self.last_time - self.first_time + span = max(duration, 1) + tasks = self.sched_tasks(filters) + io_tasks = self.io_tasks(filters) + print(f"Duration: {fmt_duration(duration)}, CPUs: {len(self.cpus)}, " + f"tasks: {len(tasks) or len(io_tasks)}, sched events: {self.sched_events}, " + f"power events: {self.power_events}, I/O events: {self.io_events}") + if not tasks_only: + for c in sorted(self.cpus.values(), key=lambda c: c.cpu): + busy = sum(e - s for s, e in zip(c.run.starts, c.run.ends)) + print(f"CPU {c.cpu}: busy {busy * 100 / span:.1f}%, " + f"{len(c.run)} runs, {len(c.cstate)} idle periods, " + f"{len(c.pstate)} frequency periods") + if power_only: + return + print(f"{'Task':<24} {'TID':>8} {'Running':>12} {'Waiting':>12} {'Blocked':>12} " + f"{'Switches':>9} {'Wakeups':>8}") + for task in sorted(tasks, key=lambda t: -t.totals[STATE_RUNNING]): + print(f"{task.comm[:24]:<24} {task.tid:>8} " + f"{fmt_duration(task.totals[STATE_RUNNING]):>12} " + f"{fmt_duration(task.totals[STATE_WAITING]):>12} " + f"{fmt_duration(task.totals[STATE_BLOCKED]):>12} " + f"{task.switches:>9} {len(task.wakeups):>8}") + for task in io_tasks: + print(f"I/O {task.name()}: {len(task.io)} syscalls, {task.io_bytes} bytes") + + +Cell = Tuple[str, Style] + + +class ThemeColors: + """Colors and Rich styles derived from the active Textual theme.""" + def __init__(self, theme_variables: Mapping[str, str]) -> None: + self.running = theme_variables["primary"] + self.waiting = theme_variables["error"] + self.blocked = theme_variables["warning"] + self.idle_light = theme_variables["secondary-lighten-2"] + self.idle_dark = theme_variables["secondary-darken-2"] + self.freq_low = theme_variables["success"] + self.freq_high = theme_variables["error"] + self.io = [ + theme_variables["success"], + theme_variables["error"], + theme_variables["warning"], + theme_variables["accent"], + theme_variables["primary"], + theme_variables["secondary-lighten-2"], + ] + run_color = Color.parse(self.running).rich_color + wait_color = Color.parse(self.waiting).rich_color + block_color = Color.parse(self.blocked).rich_color + self.run_style = Style(color=run_color) + self.wait_style = Style(color=run_color, bgcolor=wait_color) + self.block_style = Style(color=run_color, bgcolor=block_color) + low = Color.parse(self.freq_low) + high = Color.parse(self.freq_high) + self.freq_styles = [Style(color=low.blend(high, x / 8).rich_color) for x in range(9)] + light = Color.parse(self.idle_light) + dark = Color.parse(self.idle_dark) + self.idle_styles = [Style(color=light.blend(dark, x / 8).rich_color) for x in range(9)] + self.io_styles = [Style(color=Color.parse(c).rich_color) for c in self.io] + self.io_err_styles = [Style(color=Color.parse(c).rich_color, underline=True) + for c in self.io] + accent = Color.parse(theme_variables["accent"]).rich_color + accent_muted = Color.parse(theme_variables["accent-muted"]).rich_color + self.selected_style = Style(color=accent, bgcolor=accent_muted, bold=True) + + +class Row(ABC): + """A row within a timeline view.""" + def __init__(self, label: str) -> None: + self._label = label + + def label(self) -> str: + """Label shown to the left of the row.""" + return self._label + + @abstractmethod + def segments(self) -> SegmentList: + """Segments used for navigating between changes.""" + + @abstractmethod + def cells(self, t0: float, dt: float, width: int, colors: ThemeColors) -> List[Cell]: + """Cells for the columns starting at time t0 and dt wide.""" + + @abstractmethod + def describe(self, time: float, t0: float, t1: float) -> str: + """Rich markup describing the row at time within the window [t0, t1).""" + + +class TaskRow(Row): + """Row showing the running, waiting and blocked states of a task. + + The height of the bar shows the fraction of the time running, the + background shows waiting for a CPU or being blocked. + """ + def __init__(self, data: TimechartData, task: Task) -> None: + super().__init__(task.name()) + self.data = data + self.task = task + + def label(self) -> str: + return self.task.name() + + def segments(self) -> SegmentList: + return self.task.segs + + def cells(self, t0: float, dt: float, width: int, colors: ThemeColors) -> List[Cell]: + result = [] + for col in self.task.segs.coverage(t0, dt, width, NUM_STATES): + run = col[STATE_RUNNING] / dt + wait = col[STATE_WAITING] + block = col[STATE_BLOCKED] + if wait > 0 and wait >= block: + style = colors.wait_style + elif block > 0: + style = colors.block_style + else: + style = colors.run_style + result.append((bar_char(run), style)) + return result + + def describe(self, time: float, t0: float, t1: float) -> str: + task = self.task + i = task.segs.find(time) + if i >= 0: + state = task.segs.keys[i] + desc = STATE_NAMES[state] + if state == STATE_RUNNING: + desc += f" on CPU {task.segs.data[i]}" + elif state == STATE_WAITING and task.segs.data[i] >= 0: + desc += f", last ran on CPU {task.segs.data[i]}" + start = task.segs.starts[i] + end = task.segs.ends[i] + desc += (f" from {self.data.fmt_time(start)} for " + f"{fmt_duration(end - start)}") + else: + desc = "sleeping or not traced" + window = task.segs.coverage(t0, max(t1 - t0, 1), 1, NUM_STATES)[0] + span = max(t1 - t0, 1) + lines = [ + f"[b]{escape(task.name())}[/b]: {desc}", + f"In view: running {window[STATE_RUNNING] * 100 / span:.1f}%, " + f"waiting {window[STATE_WAITING] * 100 / span:.1f}%, " + f"blocked {window[STATE_BLOCKED] * 100 / span:.1f}%", + f"Total: running {fmt_duration(task.totals[STATE_RUNNING])}, " + f"waiting {fmt_duration(task.totals[STATE_WAITING])}, " + f"blocked {fmt_duration(task.totals[STATE_BLOCKED])}, " + f"{task.switches} switches, {len(task.wakeups)} wakeups", + ] + idx = bisect.bisect_right(task.wakeups, (time, sys.maxsize)) - 1 + if idx >= 0: + wake_time, waker = task.wakeups[idx] + lines.append(f"Last woken at {self.data.fmt_time(wake_time)} by " + f"{escape(self.data.task_name(waker))} (press 'w' to go to waker)") + if len(task.comms) > 1: + lines.append(f"Names: {escape(', '.join(task.comms))}") + return "\n".join(lines) + + +class CpuRow(Row): + """Row showing how busy a CPU is, colored by frequency if known.""" + def __init__(self, data: TimechartData, cpu: Cpu) -> None: + super().__init__(f"CPU {cpu.cpu}") + self.data = data + self.cpu = cpu + + def segments(self) -> SegmentList: + return self.cpu.run + + def cells(self, t0: float, dt: float, width: int, colors: ThemeColors) -> List[Cell]: + busy = self.cpu.run.coverage(t0, dt, width, 2) + freqs: Optional[List[List[float]]] = None + weighted: Optional[List[List[float]]] = None + if len(self.cpu.pstate) and self.data.max_freq > self.data.min_freq: + freqs = self.cpu.pstate.coverage(t0, dt, width, 2) + weighted = self.cpu.pstate.coverage(t0, dt, width, 2, weight_fn=float) + result = [] + for x in range(width): + style = colors.run_style + if freqs and weighted and freqs[x][1] > 0: + freq = weighted[x][1] / freqs[x][1] + frac = (freq - self.data.min_freq) / (self.data.max_freq - self.data.min_freq) + style = colors.freq_styles[max(0, min(8, round(frac * 8)))] + result.append((bar_char(busy[x][1] / dt), style)) + return result + + def describe(self, time: float, t0: float, t1: float) -> str: + c = self.cpu + i = c.run.find(time) + if i >= 0: + desc = (f"running {escape(self.data.task_name(c.run.data[i]))} from " + f"{self.data.fmt_time(c.run.starts[i])} for " + f"{fmt_duration(c.run.ends[i] - c.run.starts[i])}") + else: + desc = "idle" + i = c.cstate.find(time) + if i >= 0: + desc += f", C-state C{c.cstate.data[i]}" + i = c.pstate.find(time) + if i >= 0: + desc += f", {c.pstate.data[i] / 1000:.0f} MHz" + span = max(t1 - t0, 1) + window = c.run.coverage(t0, span, 1, 2)[0] + return "\n".join([ + f"[b]CPU {c.cpu}[/b]: {desc}", + f"In view: busy {window[1] * 100 / span:.1f}%", + "Bar height is the fraction of time busy" + + (", color is the frequency from low to high" + if len(c.pstate) else ""), + ]) + + +class CStateRow(Row): + """Row showing the idle states of a CPU.""" + def __init__(self, data: TimechartData, cpu: Cpu) -> None: + super().__init__(f" CPU {cpu.cpu} idle") + self.data = data + self.cpu = cpu + + def segments(self) -> SegmentList: + return self.cpu.cstate + + def cells(self, t0: float, dt: float, width: int, colors: ThemeColors) -> List[Cell]: + cov = self.cpu.cstate.coverage(t0, dt, width, 2) + depth = self.cpu.cstate.coverage(t0, dt, width, 2, weight_fn=float) + max_cstate = max(self.data.max_cstate, 1) + result = [] + for x in range(width): + frac = cov[x][1] / dt + style = colors.idle_styles[0] + if cov[x][1] > 0: + avg = depth[x][1] / cov[x][1] + style = colors.idle_styles[max(0, min(8, round(avg * 8 / max_cstate)))] + result.append((bar_char(frac), style)) + return result + + def describe(self, time: float, t0: float, t1: float) -> str: + c = self.cpu + i = c.cstate.find(time) + if i >= 0: + desc = (f"C{c.cstate.data[i]} from {self.data.fmt_time(c.cstate.starts[i])} for " + f"{fmt_duration(c.cstate.ends[i] - c.cstate.starts[i])}") + else: + desc = "not idle" + span = max(t1 - t0, 1) + window = c.cstate.coverage(t0, span, 1, 2)[0] + return "\n".join([ + f"[b]CPU {c.cpu} idle state[/b]: {desc}", + f"In view: idle {window[1] * 100 / span:.1f}%", + "Bar height is the fraction of time idle, darker colors are deeper C-states", + ]) + + +class FreqRow(Row): + """Row showing the frequency of a CPU.""" + def __init__(self, data: TimechartData, cpu: Cpu) -> None: + super().__init__(f" CPU {cpu.cpu} freq") + self.data = data + self.cpu = cpu + + def segments(self) -> SegmentList: + return self.cpu.pstate + + def cells(self, t0: float, dt: float, width: int, colors: ThemeColors) -> List[Cell]: + cov = self.cpu.pstate.coverage(t0, dt, width, 2) + weighted = self.cpu.pstate.coverage(t0, dt, width, 2, weight_fn=float) + max_freq = max(self.data.max_freq, 1) + min_freq = self.data.min_freq + result = [] + for x in range(width): + if cov[x][1] <= 0: + result.append((" ", colors.freq_styles[0])) + continue + freq = weighted[x][1] / cov[x][1] + frac = (freq - min_freq) / (max_freq - min_freq) if max_freq > min_freq else 1.0 + result.append((bar_char(max(freq / max_freq, 1 / 8)), + colors.freq_styles[max(0, min(8, round(frac * 8)))])) + return result + + def describe(self, time: float, t0: float, t1: float) -> str: + c = self.cpu + i = c.pstate.find(time) + if i >= 0: + desc = (f"{c.pstate.data[i] / 1000:.0f} MHz from " + f"{self.data.fmt_time(c.pstate.starts[i])} for " + f"{fmt_duration(c.pstate.ends[i] - c.pstate.starts[i])}") + else: + desc = "unknown" + return "\n".join([ + f"[b]CPU {c.cpu} frequency[/b]: {desc}", + f"Range: {self.data.min_freq / 1000:.0f} - {self.data.max_freq / 1000:.0f} MHz", + "Bar height is the frequency relative to the maximum", + ]) + + +class IoRow(Row): + """Row showing the I/O syscalls of a task.""" + def __init__(self, data: TimechartData, task: Task) -> None: + super().__init__(task.name()) + self.data = data + self.task = task + + def label(self) -> str: + return self.task.name() + + def segments(self) -> SegmentList: + return self.task.io + + def cells(self, t0: float, dt: float, width: int, colors: ThemeColors) -> List[Cell]: + errs = self.task.io.coverage(t0, dt, width, NUM_IOTYPES, + weight_fn=lambda d: 1.0 if d[1] < 0 else 0.0) + result = [] + for x, col in enumerate(self.task.io.coverage(t0, dt, width, NUM_IOTYPES)): + total = sum(col) + if total <= 0: + result.append((" ", colors.io_styles[0])) + continue + iotype = col.index(max(col)) + styles = colors.io_err_styles if sum(errs[x]) > 0 else colors.io_styles + result.append((bar_char(max(total / dt, 1 / 8)), styles[iotype])) + return result + + def describe(self, time: float, t0: float, t1: float) -> str: + task = self.task + i = task.io.find(time) + if i < 0: + i = task.io.last_before(time) + prefix = "last I/O" + else: + prefix = "in" + if i >= 0: + fd, ret = task.io.data[i] + result = f"returned {ret}" if ret >= 0 else f"failed with error {-ret}" + desc = (f"{prefix} {IOTYPE_NAMES[task.io.keys[i]]} fd={fd} at " + f"{self.data.fmt_time(task.io.starts[i])} for " + f"{fmt_duration(task.io.ends[i] - task.io.starts[i])}, {result}") + else: + desc = "no I/O" + span = max(t1 - t0, 1) + window = task.io.coverage(t0, span, 1, NUM_IOTYPES)[0] + in_view = ", ".join(f"{IOTYPE_NAMES[t]} {window[t] * 100 / span:.1f}%" + for t in range(NUM_IOTYPES) if window[t] > 0) + return "\n".join([ + f"[b]{escape(task.name())}[/b]: {desc}", + f"In view: {in_view or 'no I/O'}", + f"Total: {len(task.io)} I/O syscalls, {task.io_bytes} bytes", + ]) + + +@dataclass(frozen=True) +class TimeWindow: + """The visible time range and the cursor, shared between views. + + Immutable so that it can be a reactive value, the methods return new windows. + """ + first: int + last: int + start: float + end: float + cursor: float + + @staticmethod + def whole(first: int, last: int) -> "TimeWindow": + """A window showing the whole trace with the cursor at the start.""" + last = max(last, first + 1) + return TimeWindow(first, last, first, last, first) + + def span(self) -> float: + """Length of the visible time range.""" + return self.end - self.start + + def reset(self) -> "TimeWindow": + """Show the whole trace.""" + return replace(self, start=self.first, end=self.last) + + def with_range(self, start: float, span: float) -> "TimeWindow": + """Set the visible range clamped to the trace.""" + span = min(max(span, 100.0), self.last - self.first) + start = min(max(start, self.first), self.last - span) + return replace(self, start=start, end=start + span) + + def zoom(self, factor: float) -> "TimeWindow": + """Zoom by factor keeping the cursor at the same position.""" + cursor = self.cursor + if not self.start <= cursor <= self.end: + cursor = (self.start + self.end) / 2 + rel = (cursor - self.start) / self.span() + span = self.span() * factor + return replace(self, cursor=cursor).with_range(cursor - rel * span, span) + + def pan(self, frac: float) -> "TimeWindow": + """Pan the view by the fraction of the visible span.""" + delta = self.span() * frac + win = self.with_range(self.start + delta, self.span()) + return replace(win, cursor=min(max(self.cursor + delta, win.start), win.end)) + + def with_cursor(self, time: float) -> "TimeWindow": + """Move the cursor, scrolling to keep it visible.""" + win = replace(self, cursor=min(max(time, self.first), self.last)) + if win.cursor < win.start: + return win.with_range(win.cursor, win.span()) + if win.cursor >= win.end: + return win.with_range(win.cursor - win.span() * 0.9, win.span()) + return win + + def extend(self, first: int, last: int) -> "TimeWindow": + """Change the bounds of the trace as more of it is loaded. + + If the whole trace was visible then it still is, otherwise the visible + range is kept. + """ + last = max(last, first + 1) + whole = self.start == self.first and self.end == self.last + win = replace(self, first=first, last=last, cursor=min(max(self.cursor, first), last)) + return win.reset() if whole else win.with_range(self.start, self.span()) + + +class TimelineView(ScrollView): + """A scrollable view of rows against a time axis. + + Line 0 is a time ruler that stays at the top, the other lines are rows. + """ + BINDINGS = [ + Binding("up,k", "row_up", "Up", show=False), + Binding("down,j", "row_down", "Down", show=False), + Binding("left,h", "cursor_left", "Cursor ←", key_display="←"), + Binding("right,l", "cursor_right", "Cursor →", key_display="→"), + Binding("shift+left,H", "pan_left", "Pan left", show=False), + Binding("shift+right,L", "pan_right", "Pan right", show=False), + Binding("plus,equals_sign", "zoom_in", "Zoom in", key_display="+"), + Binding("minus", "zoom_out", "Zoom out", key_display="-"), + Binding("0,escape", "zoom_reset", "Reset zoom", key_display="0"), + Binding("n", "next_change", "Next change"), + Binding("p", "prev_change", "Prev change"), + Binding("pageup", "page_up", "Page up", show=False), + Binding("pagedown", "page_down", "Page down", show=False), + ] + + DEFAULT_CSS = """ + TimelineView { + width: 100%; + height: 1fr; + } + """ + + class SelectionChanged(Message): + """Posted when the selected row changes.""" + + # Bound to TimechartApp.window, changing it repaints the view. + window: reactive[TimeWindow] = reactive(TimeWindow.whole(0, 1)) + # Index of the selected row. + selected: reactive[int] = reactive(0) + + def __init__(self, data: TimechartData, rows: List[Row], *pos_args, **kwargs) -> None: + super().__init__(*pos_args, **kwargs) + self.can_focus = True + self.data = data + self.rows = rows + self.cache: Dict[int, List[Cell]] = {} + self.cache_key: Tuple[float, float, int, Optional[ThemeColors]] = (0.0, 0.0, 0, None) + self.label_style = Style() + self.ruler_style = Style(dim=True) + self.cursor_style = Style(reverse=True) + + def selected_row(self) -> Optional[Row]: + """The selected row, if any.""" + if 0 <= self.selected < len(self.rows): + return self.rows[self.selected] + return None + + def timeline_width(self) -> int: + """Number of columns used for the timeline.""" + return max(self.scrollable_content_region.width - LABEL_WIDTH, 1) + + def update_size(self) -> None: + """Update the virtual size after rows change or the widget resizes.""" + self.virtual_size = Size(self.scrollable_content_region.width, len(self.rows) + 1) + + def on_mount(self) -> None: + """Size the view when mounted.""" + self.update_size() + + def on_resize(self) -> None: + """Size the view when resized.""" + self.update_size() + self.refresh() + + def set_rows(self, rows: List[Row]) -> None: + """Replace the rows, after sorting or as more data is loaded. + + The selected row is kept, if it moves watch_selected scrolls to it. + """ + selected = self.selected_row() + self.rows = rows + self.cache.clear() + self.update_size() + self.selected = rows.index(selected) if selected in rows else 0 + self.refresh() + + def validate_selected(self, idx: int) -> int: + """Keep the selection within the rows.""" + return max(0, min(idx, len(self.rows) - 1)) + + def watch_selected(self) -> None: + """Keep the selection visible and tell the app.""" + self.scroll_to_selected() + self.post_message(self.SelectionChanged()) + + def scroll_to_selected(self) -> None: + """Scroll so that the selected row is visible below the ruler.""" + _, scroll_y = self.scroll_offset + visible = max(self.scrollable_content_region.height - 1, 1) + if self.selected < scroll_y: + self.scroll_to(y=self.selected, animate=False) + elif self.selected >= scroll_y + visible: + self.scroll_to(y=self.selected - visible + 1, animate=False) + + def set_window(self, window: TimeWindow) -> None: + """Change the window shared by all views, owned by the app.""" + app = self.app + if isinstance(app, TimechartApp): + app.window = window + + def ruler(self, width: int) -> Strip: + """The time axis, labeled in seconds relative to the trace start.""" + win = self.window + dt = win.span() / width + tick = 14 + decimals = max(0, min(9, 1 - math.floor(math.log10(max(dt * tick, 1) / NSEC_PER_SEC)))) + chars = [" "] * width + x = 0 + while x < width: + label = f"|{(win.start + x * dt - self.data.first_time) / NSEC_PER_SEC:.{decimals}f}" + if x + len(label) > width: + break + chars[x:x + len(label)] = list(label) + x += max(tick, len(label) + 2) + segments = [Segment(make_fixed_length_string("Time (s)", LABEL_WIDTH), self.ruler_style)] + cursor_x = self.cursor_column(width) + if 0 <= cursor_x < width: + segments.append(Segment("".join(chars[:cursor_x]), self.ruler_style)) + segments.append(Segment("▼")) + segments.append(Segment("".join(chars[cursor_x + 1:]), self.ruler_style)) + else: + segments.append(Segment("".join(chars), self.ruler_style)) + return Strip(segments) + + def cursor_column(self, width: int) -> int: + """Column of the cursor or -1 if not visible.""" + win = self.window + if not win.start <= win.cursor < win.end: + return -1 + return min(int((win.cursor - win.start) * width / win.span()), width - 1) + + def render_line(self, y: int) -> Strip: + """Render the ruler or a row.""" + width = self.timeline_width() + if y == 0: + return self.ruler(width) + _, scroll_y = self.scroll_offset + idx = scroll_y + y - 1 + if idx >= len(self.rows): + return Strip.blank(self.scrollable_content_region.width) + + colors = self.app.theme_colors if isinstance(self.app, TimechartApp) else \ + ThemeColors(self.app.theme_variables) + win = self.window + key = (win.start, win.end, width, colors) + if key != self.cache_key: + self.cache.clear() + self.cache_key = key + row = self.rows[idx] + with self.data.lock: + cells = self.cache.get(idx) + if cells is None: + cells = row.cells(win.start, win.span() / width, width, colors) + self.cache[idx] = cells + label = row.label() + + label_style = colors.selected_style if idx == self.selected else self.label_style + segments = [Segment(make_fixed_length_string(label, LABEL_WIDTH - 1) + " ", + label_style)] + cursor_x = self.cursor_column(width) + # Merge runs of cells with the same style into one segment. + text = "" + style: Optional[Style] = None + for x, (char, cell_style) in enumerate(cells): + if x == cursor_x: + cell_style = cell_style + self.cursor_style + if cell_style is not style and text: + segments.append(Segment(text, style)) + text = "" + text += char + style = cell_style + if text: + segments.append(Segment(text, style)) + return Strip(segments) + + def column_time(self, x: int) -> float: + """Time at the center of the column containing screen offset x.""" + width = self.timeline_width() + return self.window.start + (x - LABEL_WIDTH + 0.5) * self.window.span() / width + + def on_click(self, click: events.Click) -> None: + """Select the clicked row and move the cursor to the clicked time.""" + if click.x >= LABEL_WIDTH: + self.set_window(self.window.with_cursor(self.column_time(click.x))) + if click.y > 0: + _, scroll_y = self.scroll_offset + self.selected = scroll_y + click.y - 1 + + def action_row_up(self) -> None: + """Select the previous row.""" + self.selected -= 1 + + def action_row_down(self) -> None: + """Select the next row.""" + self.selected += 1 + + def action_page_up(self) -> None: + """Select a row a page up.""" + self.selected -= max(self.scrollable_content_region.height - 2, 1) + + def action_page_down(self) -> None: + """Select a row a page down.""" + self.selected += max(self.scrollable_content_region.height - 2, 1) + + def action_cursor_left(self) -> None: + """Move the cursor one column left.""" + win = self.window + self.set_window(win.with_cursor(win.cursor - win.span() / self.timeline_width())) + + def action_cursor_right(self) -> None: + """Move the cursor one column right.""" + win = self.window + self.set_window(win.with_cursor(win.cursor + win.span() / self.timeline_width())) + + def action_pan_left(self) -> None: + """Pan a quarter of the view left.""" + self.set_window(self.window.pan(-0.25)) + + def action_pan_right(self) -> None: + """Pan a quarter of the view right.""" + self.set_window(self.window.pan(0.25)) + + def action_zoom_in(self) -> None: + """Halve the visible time range around the cursor.""" + self.set_window(self.window.zoom(0.5)) + + def action_zoom_out(self) -> None: + """Double the visible time range around the cursor.""" + self.set_window(self.window.zoom(2)) + + def action_zoom_reset(self) -> None: + """Show the whole trace.""" + self.set_window(self.window.reset()) + + def action_next_change(self) -> None: + """Move the cursor to the next change in the selected row.""" + row = self.selected_row() + if row: + # Skip changes within the cursor's column. + win = self.window + with self.data.lock: + time = row.segments().next_change( + win.cursor + win.span() / self.timeline_width() / 2) + if time is not None: + self.set_window(win.with_cursor(time)) + + def action_prev_change(self) -> None: + """Move the cursor to the previous change in the selected row.""" + row = self.selected_row() + if row: + win = self.window + with self.data.lock: + time = row.segments().prev_change( + win.cursor - win.span() / self.timeline_width() / 2) + if time is not None: + self.set_window(win.with_cursor(time)) + + +class TimechartApp(App): + """A Textual application to display a timechart.""" + TITLE = "perf timechart" + + BINDINGS = [ + Binding("s", "sort", "Sort tasks", tooltip="Cycle task sort order"), + Binding("w", "goto_waker", "Go to waker", + tooltip="Select the task that last woke the selected task"), + Binding(key="^q", action="quit", description="Quit", tooltip="Quit the app"), + ] + + CSS = """ + TabbedContent, TabbedContent > ContentSwitcher, TabPane { + height: 1fr; + } + TabPane { + padding: 0; + } + .legend { + height: 1; + padding: 0 1; + } + #details { + height: 8; + border: round $primary; + padding: 0 1; + } + """ + + SORT_ORDERS = ["run time", "tid", "name", "first run"] + + # The visible time range and cursor, bound to each TimelineView's window. + window: reactive[TimeWindow] = reactive(TimeWindow.whole(0, 1), init=False) + # Index into SORT_ORDERS for the tasks view. + sort_order: reactive[int] = reactive(0, init=False) + + def __init__(self, input_name: str, power_only: bool, tasks_only: bool, + filters: Sequence[str], data: Optional[TimechartData] = None) -> None: + """Create the app, if data isn't given it is loaded from input_name. + + While the data is loading the views show what has been read so far. + """ + super().__init__() + self.input_name = input_name + self.power_only = power_only + self.tasks_only = tasks_only + self.filters = filters + # The data being displayed, possibly still being loaded. + self.data = data if data else TimechartData() + self.loaded = data is not None + # The data being loaded in a background thread. + self.loading: Optional[TimechartData] = None + self.tasks: List[Task] = [] + self.io_tasks: List[Task] = [] + # Rows keyed by type and CPU or tid. They are reused as the data loads + # so that the views can keep the selected row. + self.row_cache: Dict[Tuple[str, int], Row] = {} + self.io_tab_shown = True + # Does the summary table need updating before it is next shown? + self.summary_stale = True + self.theme_colors = ThemeColors(self.theme_variables) + + def cached_row(self, cls: Callable[[TimechartData, Any], Row], num: int, obj: Any) -> Row: + """Find or create the row of type cls for the CPU or task obj numbered num.""" + key = (getattr(cls, "__name__", ""), num) + row = self.row_cache.get(key) + if row is None: + row = cls(self.data, obj) + self.row_cache[key] = row + return row + + def cpu_rows(self) -> List[Row]: + """Rows for the CPUs tab, the data's lock must be held.""" + rows: List[Row] = [] + for cpu in sorted(self.data.cpus.values(), key=lambda c: c.cpu): + if len(cpu.run) or len(cpu.cstate) or len(cpu.pstate): + rows.append(self.cached_row(CpuRow, cpu.cpu, cpu)) + if len(cpu.cstate): + rows.append(self.cached_row(CStateRow, cpu.cpu, cpu)) + if len(cpu.pstate): + rows.append(self.cached_row(FreqRow, cpu.cpu, cpu)) + return rows + + def task_rows(self) -> List[Row]: + """Rows for the tasks tab in the current sort order, the data's lock must be held.""" + order = self.SORT_ORDERS[self.sort_order] + if order == "run time": + tasks = sorted(self.tasks, key=lambda t: -t.totals[STATE_RUNNING]) + elif order == "name": + tasks = sorted(self.tasks, key=lambda t: (t.comm, t.tid)) + elif order == "first run": + tasks = sorted(self.tasks, key=lambda t: t.segs.starts[0]) + else: + tasks = self.tasks + return [self.cached_row(TaskRow, t.tid, t) for t in tasks] + + def compose(self) -> ComposeResult: + """Composes the user interface of the application.""" + yield Header() + with TabbedContent(): + if not self.tasks_only: + with TabPane("CPUs", id="cpus"): + yield Static(id="cpus_legend", classes="legend") + yield TimelineView(self.data, [], + id="cpus_view").data_bind(TimechartApp.window) + if not self.power_only: + with TabPane("Tasks", id="tasks"): + yield Static(id="tasks_legend", classes="legend") + yield TimelineView(self.data, [], + id="tasks_view").data_bind(TimechartApp.window) + # Hidden until there are tasks doing I/O. + with TabPane("I/O", id="io"): + yield Static(id="io_legend", classes="legend") + yield TimelineView(self.data, [], + id="io_view").data_bind(TimechartApp.window) + with TabPane("Summary", id="summary"): + yield DataTable(id="summary_table", cursor_type="row") + yield Static(id="details") + yield Footer() + + def on_mount(self) -> None: + """Show the data, loading it in the background if it wasn't given.""" + self.theme_changed_signal.subscribe(self, self.on_theme_changed) + self.update_legends() + self.update_views() + view = self.active_view() + if view: + view.focus() + if self.loaded: + self.finish_loading() + else: + self.sub_title = f"Loading {self.input_name}" + self.loading = self.data + self.load_events() + + def on_theme_changed(self, _theme: Any) -> None: + """Update colors, legends and timeline views when the theme changes.""" + self.theme_colors = ThemeColors(self.theme_variables) + self.update_legends() + for view in self.query(TimelineView): + view.refresh() + + def update_legends(self) -> None: + """Update the legend strings using the active theme's colors.""" + c = self.theme_colors + with self.data.lock: + cpus_legend = self.cpu_legend() + for static in self.query("#cpus_legend").results(Static): + static.update(cpus_legend) + tasks_legend = (f"[{c.running}]█[/] running " + f"[on {c.waiting}] [/] runnable " + f"[on {c.blocked}] [/] blocked " + "(bar height is the fraction of time running)") + for static in self.query("#tasks_legend").results(Static): + static.update(tasks_legend) + io_legend = (" ".join(f"[{c.io[t]}]█[/] {IOTYPE_NAMES[t]}" + for t in range(NUM_IOTYPES)) + + " (underline is an error)") + for static in self.query("#io_legend").results(Static): + static.update(io_legend) + + @work(thread=True, exclusive=True) + def load_events(self) -> None: + """Read the perf.data file in a thread so the UI stays responsive.""" + data = self.loading or self.data + self.loading = data + + def progress() -> None: + try: + # Blocks until the UI has shown the data read so far. + self.call_from_thread(self.update_progress, data) + except RuntimeError: + # The app is no longer running. + data.cancelled = True + + data.progress = progress + data.start_progress = data.last_progress = monotonic() + try: + read_events(data, self.input_name) + except LoadCancelled: + return + except (OSError, ValueError, RuntimeError) as e: + self.call_from_thread(self.exit, None, 1, + f"Error processing {self.input_name}: {e}") + return + finally: + data.progress = None + self.loading = None + if data.cancelled: + return + if not data.has_events(): + self.call_from_thread(self.exit, None, 1, + f"Error: no scheduler, power or I/O events found in " + f"{self.input_name}.\n" + "Record them with 'perf timechart record'.") + return + self.call_from_thread(self.data_loaded) + + def data_loaded(self) -> None: + """Called on the UI thread when loading completes.""" + self.loaded = True + self.finish_loading() + + def update_progress(self, data: TimechartData) -> None: + """Show how much of the file has been processed and the data so far.""" + self.sub_title = (f"Loading {self.input_name}: {data.nr_samples:,} samples, " + f"{fmt_duration(data.last_time - data.first_time)} of trace") + self.update_views() + + def finish_loading(self) -> None: + """Show all of the data and a summary of it.""" + self.update_views() + views = self.query("#cpus_view") + if views and not views.first(TimelineView).rows: + self.query_one(TabbedContent).hide_tab("cpus") + task_views = self.query("#tasks_view") + if task_views and not task_views.first(TimelineView).rows and self.io_tasks: + self.query_one(TabbedContent).hide_tab("tasks") + nr_tasks = len(self.tasks) or len(self.io_tasks) + self.sub_title = (f"{self.input_name}: " + f"{fmt_duration(self.data.last_time - self.data.first_time)}, " + f"{len(self.data.cpus)} CPUs, {nr_tasks} tasks") + + def cancel_loading(self) -> None: + """Stop a background load, the worker notices at the next progress interval.""" + loading = self.loading + if loading is not None: + loading.cancelled = True + + async def action_quit(self) -> None: + """Quit, stopping any background load.""" + self.cancel_loading() + await super().action_quit() + + def on_unmount(self) -> None: + """Stop any background load when the app exits.""" + self.cancel_loading() + + def cpu_legend(self) -> str: + """Legend for the CPUs tab, saying which power events are missing. + + The data's lock must be held. + """ + c = self.theme_colors + data = self.data + has_pstate = any(len(cpu.pstate) for cpu in data.cpus.values()) + has_cstate = any(len(cpu.cstate) for cpu in data.cpus.values()) + parts = [f"[{c.running}]█[/] busy"] + if has_pstate and data.max_freq > data.min_freq: + parts.append(f"[{c.freq_low}]█[/]→[{c.freq_high}]█[/] frequency") + elif has_pstate: + parts.append(f"frequency constant at {data.max_freq / 1000:.0f} MHz") + if has_cstate: + parts.append(f"[{c.idle_light}]█[/]→[{c.idle_dark}]█[/] idle state") + missing = [] + if not has_pstate: + missing.append("power:cpu_frequency") + if not has_cstate: + missing.append("power:cpu_idle") + if missing: + parts.append(f"[dim](no {' or '.join(missing)} events)[/dim]") + return " ".join(parts) + + def update_views(self) -> None: + """Update the views from the data, which may still be loading.""" + data = self.data + with data.lock: + first, last = data.first_time, data.last_time + self.tasks = data.sched_tasks(self.filters) + self.io_tasks = data.io_tasks(self.filters) + all_rows = { + "cpus_view": self.cpu_rows(), + "tasks_view": self.task_rows(), + "io_view": [self.cached_row(IoRow, t.tid, t) for t in self.io_tasks], + } + legend = self.cpu_legend() + for view_id, rows in all_rows.items(): + for view in self.query(f"#{view_id}").results(TimelineView): + view.set_rows(rows) + for static in self.query("#cpus_legend").results(Static): + static.update(legend) + if self.query("#io") and bool(self.io_tasks) != self.io_tab_shown: + self.io_tab_shown = bool(self.io_tasks) + tabbed = self.query_one(TabbedContent) + if self.io_tab_shown: + tabbed.show_tab("io") + else: + tabbed.hide_tab("io") + if first or last: + self.window = self.window.extend(first, last) + self.summary_stale = True + tabs = self.query(TabbedContent) + if tabs and tabs.first(TabbedContent).active == "summary": + self.update_summary() + self.update_details() + + def update_summary(self) -> None: + """Fill in the summary table keeping the selected task.""" + tables = self.query("#summary_table") + if not tables: + return + self.summary_stale = False + table = tables.first(DataTable) + if not table.columns: + table.add_columns("Task", "TID", "Running", "Waiting", "Blocked", + "Switches", "Wakeups", "I/O bytes") + selected = None + if table.row_count: + try: + selected = table.coordinate_to_cell_key(table.cursor_coordinate).row_key + except CellDoesNotExist: + pass + summary_tasks = self.tasks if self.tasks else self.io_tasks + with self.data.lock: + rows = [(task.comm, task.tid, + fmt_duration(task.totals[STATE_RUNNING]), + fmt_duration(task.totals[STATE_WAITING]), + fmt_duration(task.totals[STATE_BLOCKED]), + task.switches, len(task.wakeups), task.io_bytes) + for task in sorted(summary_tasks, + key=lambda t: (-t.totals[STATE_RUNNING], -t.io_bytes))] + table.clear() + for row in rows: + table.add_row(*row, key=str(row[1])) + if selected is not None: + try: + table.move_cursor(row=table.get_row_index(selected), animate=False) + except RowDoesNotExist: + pass + + def active_view(self) -> Optional[TimelineView]: + """The timeline view in the active tab, if any.""" + tabs = self.query(TabbedContent) + if not tabs: + return None + active = tabs.first(TabbedContent).active + views = self.query(f"#{active}_view") if active else None + return views.first(TimelineView) if views else None + + def watch_window(self) -> None: + """The bound views repaint themselves, update the details.""" + self.update_details() + + @on(TimelineView.SelectionChanged) + def on_selection_changed(self) -> None: + """Describe the newly selected row.""" + self.update_details() + + def update_details(self) -> None: + """Describe the selected row at the cursor.""" + details = self.query("#details") + if not details: + return + win = self.window + text = (f"Cursor [b]{self.data.fmt_time(win.cursor)}[/b] " + f"View {self.data.fmt_time(win.start)} - {self.data.fmt_time(win.end)} " + f"({fmt_duration(win.span())})\n") + view = self.active_view() + row = view.selected_row() if view else None + if row: + with self.data.lock: + text += row.describe(win.cursor, win.start, win.end) + elif view is None: + text += "Select a task and press enter to show it in the Tasks timeline" + details.first(Static).update(text) + + @on(TabbedContent.TabActivated) + def on_tab_activated(self) -> None: + """Focus the view in the newly active tab.""" + view = self.active_view() + if view: + view.focus() + else: + if self.summary_stale: + self.update_summary() + tables = self.query("#summary_table") + if tables: + tables.first(DataTable).focus() + self.update_details() + + @on(DataTable.RowSelected) + def on_row_selected(self, event: DataTable.RowSelected) -> None: + """Show the selected summary task in the tasks timeline.""" + if event.row_key.value is not None: + self.goto_task(int(event.row_key.value), None) + + def goto_task(self, tid: int, time: Optional[float]) -> None: + """Select the task in the tasks or I/O view, optionally moving the cursor.""" + for tab_id, view_id in (("tasks", "#tasks_view"), ("io", "#io_view")): + views = self.query(view_id) + if not views: + continue + view = views.first(TimelineView) + for idx, row in enumerate(view.rows): + if isinstance(row, (TaskRow, IoRow)) and row.task.tid == tid: + self.query_one(TabbedContent).active = tab_id + if time is not None: + self.window = self.window.with_cursor(time) + view.selected = idx + view.focus() + return + self.notify(f"Task {tid} isn't shown in the tasks view", severity="warning") + + def action_sort(self) -> None: + """Cycle the sort order of the tasks view.""" + self.sort_order = (self.sort_order + 1) % len(self.SORT_ORDERS) + + def watch_sort_order(self) -> None: + """Re-sort the tasks view.""" + views = self.query("#tasks_view") + if not views: + return + with self.data.lock: + rows = self.task_rows() + views.first(TimelineView).set_rows(rows) + self.notify(f"Tasks sorted by {self.SORT_ORDERS[self.sort_order]}") + self.update_details() + + def action_goto_waker(self) -> None: + """Select the task that woke the selected task before the cursor.""" + view = self.active_view() + row = view.selected_row() if view else None + if not isinstance(row, (TaskRow, IoRow)): + return + with self.data.lock: + wakeups = row.task.wakeups + idx = bisect.bisect_right(wakeups, (self.window.cursor, sys.maxsize)) - 1 + wake_time, waker = wakeups[idx] if idx >= 0 else (0, 0) + if idx < 0: + self.notify("No wakeup before the cursor", severity="warning") + return + if waker <= 0: + self.notify(f"Woken by {self.data.task_name(waker)} at " + f"{self.data.fmt_time(wake_time)}") + return + self.goto_task(waker, wake_time) + + +def main() -> None: + """Parse arguments, read the perf.data file and run the app.""" + parser = argparse.ArgumentParser( + description="Interactive timechart of CPU, task and I/O activity.") + parser.add_argument("-i", "--input", default="perf.data", help="input perf.data file") + parser.add_argument("-P", "--power-only", action="store_true", + help="only show CPU power information") + parser.add_argument("-T", "--tasks-only", action="store_true", + help="only show task information") + parser.add_argument("-p", "--process", action="append", default=[], + help="only show processes with the given name or PID, may be repeated") + parser.add_argument("--dump", action="store_true", + help="print a text summary rather than running the interactive UI") + args = parser.parse_args() + + if args.power_only and args.tasks_only: + print("Error: -P and -T options cannot be used at the same time.", file=sys.stderr) + sys.exit(1) + + if args.input == "-": + if not args.dump: + # The interactive UI reads the keyboard from stdin. + print("Error: reading perf.data from stdin requires --dump.", file=sys.stderr) + sys.exit(1) + elif not os.path.exists(args.input): + print(f"Error: {args.input} not found. (try 'perf timechart record' first)", + file=sys.stderr) + sys.exit(1) + + if args.dump: + data = load_data(args.input) + data.dump(args.power_only, args.tasks_only, args.process) + return + + # The app starts immediately and loads the data in the background. + app = TimechartApp(args.input, args.power_only, args.tasks_only, args.process) + app.run() + sys.exit(app.return_code or 0) + + +def read_events(data: TimechartData, input_name: str) -> None: + """Process the events in input_name into data, raising on errors.""" + if data.cancelled: + raise LoadCancelled() + try: + data.session = perf.session(perf.data(input_name), sample=data.process_event) + data.session.process_events() + data.finish() + finally: + # Break the reference cycle between the session and the callback. + data.session = None + + +def load_data(input_name: str) -> TimechartData: + """Read the perf.data file exiting on errors or if there's nothing to show.""" + if input_name != "-" and not os.path.exists(input_name): + print(f"Error: {input_name} not found. (try 'perf timechart record' first)", + file=sys.stderr) + sys.exit(1) + + data = TimechartData() + try: + read_events(data, input_name) + except (OSError, ValueError, RuntimeError) as e: + print(f"Error processing {input_name}: {e}", file=sys.stderr) + sys.exit(1) + except KeyboardInterrupt: + data.finish() + + if not data.has_events(): + print(f"Error: no scheduler, power or I/O events found in {input_name}.\n" + "Record them with 'perf timechart record'.", file=sys.stderr) + sys.exit(1) + return data + + +if __name__ == "__main__": + main() diff --git a/tools/perf/python/twatch.py b/tools/perf/python/twatch.py index 04f3db29b9bc..b0f1bf113162 100755 --- a/tools/perf/python/twatch.py +++ b/tools/perf/python/twatch.py @@ -1,5 +1,6 @@ #! /usr/bin/env python # SPDX-License-Identifier: GPL-2.0-only +"""Example to show how to enable a software event and track context switches.""" # -*- python -*- # -*- coding: utf-8 -*- # twatch - Experimental use of the perf python interface @@ -9,53 +10,51 @@ import perf def main(context_switch = 0, thread = -1): - cpus = perf.cpu_map() - threads = perf.thread_map(thread) - evsel = perf.evsel(type = perf.TYPE_SOFTWARE, - config = perf.COUNT_SW_DUMMY, - task = 1, comm = 1, mmap = 0, freq = 0, - wakeup_events = 1, watermark = 1, - sample_id_all = 1, context_switch = context_switch, - sample_type = perf.SAMPLE_PERIOD | perf.SAMPLE_TID | perf.SAMPLE_CPU) - - """What we want are just the PERF_RECORD_ lifetime events for threads, - using the default, PERF_TYPE_HARDWARE + PERF_COUNT_HW_CYCLES & freq=1 - (the default), makes perf reenable irq_vectors:local_timer_entry, when - disabling nohz, not good for some use cases where all we want is to get - threads comes and goes... So use (perf.TYPE_SOFTWARE, perf_COUNT_SW_DUMMY, - freq=0) instead.""" - - evsel.open(cpus = cpus, threads = threads); - evlist = perf.evlist(cpus, threads) - evlist.add(evsel) - evlist.mmap() - while True: - evlist.poll(timeout = -1) - for cpu in cpus: - event = evlist.read_on_cpu(cpu) - if not event: - continue - print("cpu: {0}, pid: {1}, tid: {2} {3}".format(event.sample_cpu, + cpus = perf.cpu_map() + threads = perf.thread_map(thread) + evsel = perf.evsel(type = perf.TYPE_SOFTWARE, + config = perf.COUNT_SW_DUMMY, + task = 1, comm = 1, mmap = 0, freq = 0, + wakeup_events = 1, watermark = 1, + sample_id_all = 1, context_switch = context_switch, + sample_type = perf.SAMPLE_PERIOD | perf.SAMPLE_TID | perf.SAMPLE_CPU) + + # What we want are just the PERF_RECORD_ lifetime events for threads, + # using the default, PERF_TYPE_HARDWARE + PERF_COUNT_HW_CYCLES & freq=1 + # (the default), makes perf reenable irq_vectors:local_timer_entry, when + # disabling nohz, not good for some use cases where all we want is to get + # threads comes and goes... So use (perf.TYPE_SOFTWARE, perf_COUNT_SW_DUMMY, + # freq=0) instead. + + evsel.open(cpus = cpus, threads = threads) + evlist = perf.evlist(cpus, threads) + evlist.add(evsel) + evlist.mmap() + while True: + evlist.poll(timeout = -1) + for cpu in cpus: + event = evlist.read_on_cpu(cpu) + if not event: + continue + print("cpu: {0}, pid: {1}, tid: {2} {3}".format(event.sample_cpu, event.sample_pid, event.sample_tid, event)) if __name__ == '__main__': - """ - To test the PERF_RECORD_SWITCH record, pick a pid and replace - in the following line. + # To test the PERF_RECORD_SWITCH record, pick a pid and replace + # in the following line. - Example output: + # Example output: -cpu: 3, pid: 31463, tid: 31593 { type: context_switch, next_prev_pid: 31463, next_prev_tid: 31593, switch_out: 1 } -cpu: 1, pid: 31463, tid: 31489 { type: context_switch, next_prev_pid: 31463, next_prev_tid: 31489, switch_out: 1 } -cpu: 2, pid: 31463, tid: 31496 { type: context_switch, next_prev_pid: 31463, next_prev_tid: 31496, switch_out: 1 } -cpu: 3, pid: 31463, tid: 31491 { type: context_switch, next_prev_pid: 31463, next_prev_tid: 31491, switch_out: 0 } + # cpu: 3, pid: 31463, tid: 31593 { type: context_switch, next_prev_pid: 31463, next_prev_tid: 31593, switch_out: 1 } + # cpu: 1, pid: 31463, tid: 31489 { type: context_switch, next_prev_pid: 31463, next_prev_tid: 31489, switch_out: 1 } + # cpu: 2, pid: 31463, tid: 31496 { type: context_switch, next_prev_pid: 31463, next_prev_tid: 31496, switch_out: 1 } + # cpu: 3, pid: 31463, tid: 31491 { type: context_switch, next_prev_pid: 31463, next_prev_tid: 31491, switch_out: 0 } + # + # It is possible as well to use event.misc & perf.PERF_RECORD_MISC_SWITCH_OUT + # to figure out if this is a context switch in or out of the monitored threads. - It is possible as well to use event.misc & perf.PERF_RECORD_MISC_SWITCH_OUT - to figure out if this is a context switch in or out of the monitored threads. - - If bored, please add command line option parsing support for these options :-) - """ - # main(context_switch = 1, thread = 31463) + # If bored, please add command line option parsing support for these options :-) + # # main(context_switch = 1, thread = 31463) main() diff --git a/tools/perf/python/wakeup-latency.py b/tools/perf/python/wakeup-latency.py new file mode 100755 index 000000000000..f77045d02734 --- /dev/null +++ b/tools/perf/python/wakeup-latency.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: GPL-2.0-only +"""Display avg/min/max wakeup latency.""" +from __future__ import annotations + +import argparse +from collections import defaultdict +import sys +from typing import Optional, Dict +import perf + +class WakeupLatency: + """Tracks and displays wakeup latency statistics.""" + def __init__(self) -> None: + self.last_wakeup: Dict[int, int] = defaultdict(int) + self.max_wakeup_latency: int = 0 + self.min_wakeup_latency: Optional[int] = None + self.total_wakeup_latency = 0 + self.total_wakeups = 0 + self.unhandled: Dict[str, int] = defaultdict(int) + self.session: Optional[perf.session] = None + + def process_event(self, sample: perf.sample_event) -> None: + """Process events.""" + event_name = str(sample.evsel) + if event_name.startswith("evsel(") and event_name.endswith(")"): + event_name = event_name[6:-1] + sample_time = sample.sample_time + + if "sched:sched_wakeup" in event_name: + try: + pid = sample.pid + self.last_wakeup[pid] = sample_time + except AttributeError: + self.unhandled[event_name] += 1 + elif "sched:sched_switch" in event_name: + try: + next_pid = sample.next_pid + wakeup_ts = self.last_wakeup.get(next_pid, 0) + if wakeup_ts: + latency = sample_time - wakeup_ts + self.max_wakeup_latency = max(self.max_wakeup_latency, latency) + if self.min_wakeup_latency is None: + self.min_wakeup_latency = latency + else: + self.min_wakeup_latency = min(self.min_wakeup_latency, latency) + self.total_wakeup_latency += latency + self.total_wakeups += 1 + del self.last_wakeup[next_pid] + except AttributeError: + self.unhandled[event_name] += 1 + else: + self.unhandled[event_name] += 1 + + def print_totals(self) -> None: + """Print summary statistics.""" + print("wakeup_latency stats:\n") + print(f"total_wakeups: {self.total_wakeups}") + if self.total_wakeups: + avg = self.total_wakeup_latency // self.total_wakeups + print(f"avg_wakeup_latency (ns): {avg}") + print(f"min_wakeup_latency (ns): {self.min_wakeup_latency}") + print(f"max_wakeup_latency (ns): {self.max_wakeup_latency}") + else: + print("avg_wakeup_latency (ns): N/A") + print("min_wakeup_latency (ns): N/A") + print("max_wakeup_latency (ns): N/A") + + if self.unhandled: + print("\nunhandled events:\n") + print(f"{'event':<40s} {'count':>10s}") + print(f"{'-'*40} {'-'*10}") + for event_name, count in self.unhandled.items(): + print(f"{event_name:<40s} {count:10d}") + + def run(self, input_file: str) -> None: + """Run the session.""" + self.session = perf.session(perf.data(input_file), sample=self.process_event) + try: + self.session.process_events() + except KeyboardInterrupt: + pass + finally: + # Break the reference cycle between self.session and the bound + # self.process_event callback so the C perf.session object is freed. + self.session = None + self.print_totals() + +def main() -> None: + """Main function.""" + parser = argparse.ArgumentParser(description="Trace wakeup latency") + parser.add_argument("-i", "--input", default="perf.data", help="Input file") + args = parser.parse_args() + + analyzer = WakeupLatency() + try: + analyzer.run(args.input) + except IOError as e: + print(e, file=sys.stderr) + sys.exit(1) + +if __name__ == "__main__": + main() |