From c897328344bedf5fa26a88e94ea272d5b348f8d5 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Tue, 26 Sep 2023 14:21:25 -0400 Subject: [PATCH 1/8] gh-110019: Refactor summarize_stats This refactors summarize_stats so that the comparative tables are easier to make and use more common code. --- Tools/scripts/summarize_stats.py | 1636 ++++++++++++++++-------------- 1 file changed, 857 insertions(+), 779 deletions(-) diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index bdca51df3dac53..103d3be6a7c86b 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -2,910 +2,988 @@ default stats folders. """ +from __future__ import annotations + # NOTE: Bytecode introspection modules (opcode, dis, etc.) should only -# happen when loading a single dataset. When comparing datasets, it +# be imported when loading a single dataset. When comparing datasets, it # could get it wrong, leading to subtle errors. import argparse import collections -import json -import os.path from datetime import date import itertools -import sys +import json +import os +from pathlib import Path import re +import sys +from typing import TextIO, TypeAlias + + +OpcodeStats: TypeAlias = dict[str, dict[str, int]] +PairCounts: TypeAlias = list[tuple[int, tuple[str, str]]] +Defines: TypeAlias = dict[int, list[str]] +Rows: TypeAlias = list[tuple] +Columns: TypeAlias = tuple[str, ...] + if os.name == "nt": DEFAULT_DIR = "c:\\temp\\py_stats\\" else: DEFAULT_DIR = "/tmp/py_stats/" -TOTAL = "specialization.hit", "specialization.miss", "execution_count" +SOURCE_DIR = Path(__file__).parents[2] -def format_ratio(num, den): - """ - Format a ratio as a percentage. When the denominator is 0, returns the empty - string. - """ - if den == 0: - return "" - else: - return f"{num/den:.01%}" +TOTAL = "specialization.hit", "specialization.miss", "execution_count" -def percentage_to_float(s): - """ - Converts a percentage string to a float. The empty string is returned as 0.0 - """ - if s == "": - return 0.0 - else: - assert s[-1] == "%" - return float(s[:-1]) +def pretty(name: str) -> str: + return name.replace("_", " ").lower() -def join_rows(a_rows, b_rows): - """ - Joins two tables together, side-by-side, where the first column in each is a - common key. - """ - if len(a_rows) == 0 and len(b_rows) == 0: - return [] - if len(a_rows): - a_ncols = list(set(len(x) for x in a_rows)) - if len(a_ncols) != 1: - raise ValueError("Table a is ragged") +class Stats(dict): + def __init__(self, input: Path): + super().__init__() - if len(b_rows): - b_ncols = list(set(len(x) for x in b_rows)) - if len(b_ncols) != 1: - raise ValueError("Table b is ragged") + if input.is_file(): + with open(input, "r") as fd: + self.update(json.load(fd)) - if len(a_rows) and len(b_rows) and a_ncols[0] != b_ncols[0]: - raise ValueError("Tables have different widths") + self["_stats_defines"] = { + int(k): v for k, v in self["_stats_defines"].items() + } + self["_defines"] = {int(k): v for k, v in self["_defines"].items()} - if len(a_rows): - ncols = a_ncols[0] - else: - ncols = b_ncols[0] + elif input.is_dir(): + stats: collections.Counter = collections.Counter() + for filename in input.iterdir(): + with open(filename) as fd: + for line in fd: + try: + key, value = line.split(":") + except ValueError: + print( + f"Unparsable line: '{line.strip()}' in {filename}", + file=sys.stderr, + ) + continue + stats[key.strip()] += int(value) + stats["__nfiles__"] += 1 - default = [""] * (ncols - 1) - a_data = {x[0]: x[1:] for x in a_rows} - b_data = {x[0]: x[1:] for x in b_rows} + self.update(stats) - if len(a_data) != len(a_rows) or len(b_data) != len(b_rows): - raise ValueError("Duplicate keys") + self._load_metadata_from_source() - # To preserve ordering, use A's keys as is and then add any in B that aren't - # in A - keys = list(a_data.keys()) + [k for k in b_data.keys() if k not in a_data] - return [(k, *a_data.get(k, default), *b_data.get(k, default)) for k in keys] + else: + raise ValueError(f"{input:r} is not a file or directory path") + + self._opcode_stats: dict[str, OpcodeStats] = {} + + def _load_metadata_from_source(self): + def get_defines(filepath: Path, prefix: str = "SPEC_FAIL") -> Defines: + with open(SOURCE_DIR / filepath) as spec_src: + defines = collections.defaultdict(list) + start = "#define " + prefix + "_" + for line in spec_src: + line = line.strip() + if not line.startswith(start): + continue + line = line[len(start) :] + name, val = line.split() + defines[int(val.strip())].append(name.strip()) + return defines + import opcode -def calculate_specialization_stats(family_stats, total): - rows = [] - for key in sorted(family_stats): - if key.startswith("specialization.failure_kinds"): - continue - if key in ("specialization.hit", "specialization.miss"): - label = key[len("specialization.") :] - elif key == "execution_count": - continue - elif key in ( - "specialization.success", - "specialization.failure", - "specializable", - ): - continue - elif key.startswith("pair"): - continue - else: - label = key - rows.append( - ( - f"{label:>12}", - f"{family_stats[key]:>12}", - format_ratio(family_stats[key], total), - ) - ) - return rows - - -def calculate_specialization_success_failure(family_stats): - total_attempts = 0 - for key in ("specialization.success", "specialization.failure"): - total_attempts += family_stats.get(key, 0) - rows = [] - if total_attempts: - for key in ("specialization.success", "specialization.failure"): - label = key[len("specialization.") :] - label = label[0].upper() + label[1:] - val = family_stats.get(key, 0) - rows.append((label, val, format_ratio(val, total_attempts))) - return rows - - -def calculate_specialization_failure_kinds(name, family_stats, defines): - total_failures = family_stats.get("specialization.failure", 0) - failure_kinds = [0] * 40 - for key in family_stats: - if not key.startswith("specialization.failure_kind"): - continue - _, index = key[:-1].split("[") - index = int(index) - failure_kinds[index] = family_stats[key] - failures = [(value, index) for (index, value) in enumerate(failure_kinds)] - failures.sort(reverse=True) - rows = [] - for value, index in failures: - if not value: - continue - rows.append( - ( - kind_to_text(index, defines, name), - value, - format_ratio(value, total_failures), - ) + self["_specialized_instructions"] = [ + op + for op in opcode._specialized_opmap.keys() # type: ignore + if "__" not in op + ] + self["_stats_defines"] = get_defines( + Path("Include") / "cpython" / "pystats.h", "EVAL_CALL" ) - return rows + self["_defines"] = get_defines(Path("Python") / "specialize.c") + @property + def defines(self) -> Defines: + return self["_defines"] -def print_specialization_stats(name, family_stats, defines): - if "specializable" not in family_stats: - return - total = sum(family_stats.get(kind, 0) for kind in TOTAL) - if total == 0: - return - with Section(name, 3, f"specialization stats for {name} family"): - rows = calculate_specialization_stats(family_stats, total) - emit_table(("Kind", "Count", "Ratio"), rows) - rows = calculate_specialization_success_failure(family_stats) - if rows: - print_title("Specialization attempts", 4) - emit_table(("", "Count:", "Ratio:"), rows) - rows = calculate_specialization_failure_kinds(name, family_stats, defines) - emit_table(("Failure kind", "Count:", "Ratio:"), rows) - - -def print_comparative_specialization_stats( - name, base_family_stats, head_family_stats, defines -): - if "specializable" not in base_family_stats: - return + @property + def pystats_defines(self) -> Defines: + return self["_stats_defines"] - base_total = sum(base_family_stats.get(kind, 0) for kind in TOTAL) - head_total = sum(head_family_stats.get(kind, 0) for kind in TOTAL) - if base_total + head_total == 0: - return - with Section(name, 3, f"specialization stats for {name} family"): - base_rows = calculate_specialization_stats(base_family_stats, base_total) - head_rows = calculate_specialization_stats(head_family_stats, head_total) - emit_table( - ("Kind", "Base Count", "Base Ratio", "Head Count", "Head Ratio"), - join_rows(base_rows, head_rows), + @property + def specialized_instructions(self) -> list[str]: + return self["_specialized_instructions"] + + def get_opcode_stats(self, prefix: str) -> OpcodeStats: + if prefix in self._opcode_stats: + return self._opcode_stats[prefix] + + opcode_stats: OpcodeStats = collections.defaultdict(dict) + for key, value in self.items(): + if not key.startswith(prefix): + continue + name, _, rest = key[len(prefix) + 1 :].partition("]") + opcode_stats[name][rest.strip(".")] = value + + self._opcode_stats[prefix] = opcode_stats + return opcode_stats + + def get_pair_counts(self, prefix: str) -> PairCounts: + opcode_stats = self.get_opcode_stats(prefix) + pair_counts: PairCounts = [] + for name_i, opcode_stat in opcode_stats.items(): + for key, value in opcode_stat.items(): + if key.startswith("pair_count"): + name_j, _, _ = key[len("pair_count") + 1 :].partition("]") + if value: + pair_counts.append((value, (name_i, name_j))) + pair_counts.sort(reverse=True) + return pair_counts + + def get_total(self, prefix: str) -> int: + return sum( + x.get("execution_count", 0) for x in self.get_opcode_stats(prefix).values() ) - base_rows = calculate_specialization_success_failure(base_family_stats) - head_rows = calculate_specialization_success_failure(head_family_stats) - rows = join_rows(base_rows, head_rows) - if rows: - print_title("Specialization attempts", 4) - emit_table( - ("", "Base Count:", "Base Ratio:", "Head Count:", "Head Ratio:"), rows - ) - base_rows = calculate_specialization_failure_kinds( - name, base_family_stats, defines - ) - head_rows = calculate_specialization_failure_kinds( - name, head_family_stats, defines - ) - emit_table( - ( - "Failure kind", - "Base Count:", - "Base Ratio:", - "Head Count:", - "Head Ratio:", - ), - join_rows(base_rows, head_rows), - ) -def gather_stats(input): - # Note the output of this function must be JSON-serializable - - if os.path.isfile(input): - with open(input, "r") as fd: - stats = json.load(fd) - - stats["_stats_defines"] = { - int(k): v for k, v in stats["_stats_defines"].items() - } - stats["_defines"] = {int(k): v for k, v in stats["_defines"].items()} - return stats - - elif os.path.isdir(input): - stats = collections.Counter() - for filename in os.listdir(input): - with open(os.path.join(input, filename)) as fd: - for line in fd: - try: - key, value = line.split(":") - except ValueError: - print( - f"Unparsable line: '{line.strip()}' in {filename}", - file=sys.stderr, - ) - continue - key = key.strip() - value = int(value) - stats[key] += value - stats["__nfiles__"] += 1 +class Count(int): + def markdown(self) -> str: + return format(self, ",d") - import opcode - stats["_specialized_instructions"] = [ - op for op in opcode._specialized_opmap.keys() if "__" not in op - ] - stats["_stats_defines"] = get_stats_defines() - stats["_defines"] = get_defines() +class CountPer: + def __init__(self, num: int, den: int): + self.num = num + self.den = den + if den == 0 and num != 0: + raise ValueError("Invalid denominator") - return stats - else: - raise ValueError(f"{input:r} is not a file or directory path") + def markdown(self) -> str: + if self.den == 0: + return "0" + else: + return f"{int(self.num / self.den):,d}" -def extract_opcode_stats(stats, prefix): - opcode_stats = collections.defaultdict(dict) - for key, value in stats.items(): - if not key.startswith(prefix): - continue - name, _, rest = key[len(prefix) + 1 :].partition("]") - opcode_stats[name][rest.strip(".")] = value - return opcode_stats +class Ratio: + def __init__(self, num: int, den: int): + self.num = num + self.den = den + def __float__(self): + if self.den == 0: + return 0.0 + else: + return self.num / self.den -def parse_kinds(spec_src, prefix="SPEC_FAIL"): - defines = collections.defaultdict(list) - start = "#define " + prefix + "_" - for line in spec_src: - line = line.strip() - if not line.startswith(start): - continue - line = line[len(start) :] - name, val = line.split() - defines[int(val.strip())].append(name.strip()) - return defines - - -def pretty(defname): - return defname.replace("_", " ").lower() - - -def kind_to_text(kind, defines, opname): - if kind <= 8: - return pretty(defines[kind][0]) - if opname == "LOAD_SUPER_ATTR": - opname = "SUPER" - elif opname.endswith("ATTR"): - opname = "ATTR" - elif opname in ("FOR_ITER", "SEND"): - opname = "ITER" - elif opname.endswith("SUBSCR"): - opname = "SUBSCR" - for name in defines[kind]: - if name.startswith(opname): - return pretty(name[len(opname) + 1 :]) - return "kind " + str(kind) - - -def categorized_counts(opcode_stats, specialized_instructions): - basic = 0 - specialized = 0 - not_specialized = 0 - for name, opcode_stat in opcode_stats.items(): - if "execution_count" not in opcode_stat: - continue - count = opcode_stat["execution_count"] - if "specializable" in opcode_stat: - not_specialized += count - elif name in specialized_instructions: - miss = opcode_stat.get("specialization.miss", 0) - not_specialized += miss - specialized += count - miss + def markdown(self) -> str: + if self.den == 0: + return "" else: - basic += count - return basic, not_specialized, specialized + return f"{self.num / self.den:,.01%}" -def print_title(name, level=2): - print("#" * level, name) - print() +class DiffRatio(Ratio): + def __init__(self, base: int | str, head: int | str): + if isinstance(base, str) or isinstance(head, str): + super().__init__(0, 0) + else: + super().__init__(head - base, base) -class Section: - def __init__(self, title, level=2, summary=None): - self.title = title - self.level = level - if summary is None: - self.summary = title.lower() - else: - self.summary = summary +def sort_by_last_column(rows: Rows): + rows.sort(key=lambda row: abs(float(row[-1])), reverse=True) - def __enter__(self): - print_title(self.title, self.level) - print("
") - print("", self.summary, "") - print() - return self - def __exit__(*args): - print() - print("
") - print() +def dont_sort(rows: Rows): + pass -def to_str(x): - if isinstance(x, int): - return format(x, ",d") - else: - return str(x) - - -def emit_table(header, rows): - width = len(header) - header_line = "|" - under_line = "|" - for item in header: - under = "---" - if item.endswith(":"): - item = item[:-1] - under += ":" - header_line += item + " | " - under_line += under + "|" - print(header_line) - print(under_line) - for row in rows: - if width is not None and len(row) != width: - raise ValueError("Wrong number of elements in row '" + str(row) + "'") - print("|", " | ".join(to_str(i) for i in row), "|") - print() - - -def emit_histogram(title, stats, key, total): - rows = [] - for k, v in stats.items(): - if k.startswith(key): - entry = int(re.match(r".+\[([0-9]+)\]", k).groups()[0]) - rows.append((f"<= {entry}", int(v), format_ratio(int(v), total))) - # Don't include larger buckets with 0 entries - for j in range(len(rows) - 1, -1, -1): - if rows[j][1] != 0: - break - rows = rows[: j + 1] - - print(f"**{title}**\n") - emit_table(("Range", "Count:", "Ratio:"), rows) - - -def calculate_execution_counts(opcode_stats, total): - counts = [] - for name, opcode_stat in opcode_stats.items(): - if "execution_count" in opcode_stat: - count = opcode_stat["execution_count"] - miss = 0 - if "specializable" not in opcode_stat: - miss = opcode_stat.get("specialization.miss") - counts.append((count, name, miss)) - counts.sort(reverse=True) - cumulative = 0 - rows = [] - for count, name, miss in counts: - cumulative += count - if miss: - miss = format_ratio(miss, count) - else: - miss = "" - rows.append( - ( - name, - count, - format_ratio(count, total), - format_ratio(cumulative, total), - miss, - ) +class Table: + columns: Columns + sort_by_last_column: bool = False + + def calculate_rows(self, stats: Stats) -> Rows: + raise NotImplementedError() + + def join_row(self, key: str, row_a: tuple, row_b: tuple) -> tuple: + return (key, *row_a, *row_b) + + def join_columns(self, columns: Columns) -> Columns: + return ( + columns[0], + *("Base " + x for x in columns[1:]), + *("Head " + x for x in columns[1:]), ) - return rows + sort_joined_rows = staticmethod(dont_sort) -def emit_execution_counts(opcode_stats, total): - with Section("Execution counts", summary="execution counts for all instructions"): - rows = calculate_execution_counts(opcode_stats, total) - emit_table(("Name", "Count:", "Self:", "Cumulative:", "Miss ratio:"), rows) + def join_tables(self, rows_a: Rows, rows_b: Rows) -> tuple[Columns, Rows]: + ncols = len(self.columns) + default = ("",) * (ncols - 1) + data_a = {x[0]: x[1:] for x in rows_a} + data_b = {x[0]: x[1:] for x in rows_b} -def _emit_comparative_execution_counts(base_rows, head_rows): - base_data = {x[0]: x[1:] for x in base_rows} - head_data = {x[0]: x[1:] for x in head_rows} - opcodes = base_data.keys() | head_data.keys() + if len(data_a) != len(rows_a) or len(data_b) != len(rows_b): + raise ValueError("Duplicate keys") - rows = [] - default = [0, "0.0%", "0.0%", 0] - for opcode in opcodes: - base_entry = base_data.get(opcode, default) - head_entry = head_data.get(opcode, default) - if base_entry[0] == 0: - change = 1 + # To preserve ordering, use A's keys as is and then add any in B that + # aren't in A + keys = list(data_a.keys()) + [k for k in data_b.keys() if k not in data_a] + rows = [ + self.join_row(k, data_a.get(k, default), data_b.get(k, default)) + for k in keys + ] + if self.sort_by_last_column: + rows.sort(key=lambda row: abs(float(row[-1])), reverse=True) + + columns = self.join_columns(self.columns) + return columns, rows + + def get_rows( + self, base_stats: Stats, head_stats: Stats | None = None + ) -> tuple[Columns, Rows]: + if head_stats is None: + rows = self.calculate_rows(base_stats) + return self.columns, rows else: - change = (head_entry[0] - base_entry[0]) / base_entry[0] - rows.append((opcode, base_entry[0], head_entry[0], f"{change:0.1%}")) + rows_a = self.calculate_rows(base_stats) + rows_b = self.calculate_rows(head_stats) + cols, rows = self.join_tables(rows_a, rows_b) + return cols, rows + + def output_markdown( + self, + out: TextIO, + base_stats: Stats, + head_stats: Stats | None = None, + level: int = 2, + ) -> None: + header, rows = self.get_rows(base_stats, head_stats) + if len(rows) == 0: + return + + def to_markdown(x): + if hasattr(x, "markdown"): + return x.markdown() + elif isinstance(x, str): + return x + elif x is None: + return "" + else: + raise TypeError(f"Can't convert {x} to markdown") + + width = len(header) + header_line = "|" + under_line = "|" + for item in header: + under = "---" + if item.endswith(":"): + item = item[:-1] + under += ":" + header_line += item + " | " + under_line += under + "|" + print(header_line, file=out) + print(under_line, file=out) + for row in rows: + if width is not None and len(row) != width: + raise ValueError("Wrong number of elements in row '" + str(row) + "'") + print("|", " | ".join(to_markdown(i) for i in row), "|", file=out) + print(file=out) + + +class FixedTable(Table): + def __init__(self, columns: Columns, rows: Rows): + self.columns = columns + self.rows = rows + + def get_rows(self, *args) -> tuple[Columns, Rows]: + return self.columns, self.rows + + +class SimpleChangeTable(Table): + """ + Base class of tables where the comparison table has an extra column "Change" + computed from the change of the second column of the base and head. Sorted + by the "Change" column. + """ - rows.sort(key=lambda x: abs(percentage_to_float(x[-1])), reverse=True) + def join_row(self, key: str, base_data: tuple, head_data: tuple) -> tuple: + return (key, *base_data, *head_data, DiffRatio(base_data[0], head_data[0])) - emit_table(("Name", "Base Count:", "Head Count:", "Change:"), rows) + def join_columns(self, columns: Columns) -> Columns: + return super().join_columns(columns) + ("Change:",) + sort_joined_rows = staticmethod(sort_by_last_column) -def emit_comparative_execution_counts( - base_opcode_stats, base_total, head_opcode_stats, head_total, level=2 -): - with Section( - "Execution counts", summary="execution counts for all instructions", level=level - ): - base_rows = calculate_execution_counts(base_opcode_stats, base_total) - head_rows = calculate_execution_counts(head_opcode_stats, head_total) - _emit_comparative_execution_counts(base_rows, head_rows) +class ExecutionCountTable(Table): + columns = ("Name", "Count:", "Self:", "Cumulative:", "Miss ratio:") + + def __init__(self, prefix: str): + self.prefix = prefix -def get_defines(): - spec_path = os.path.join(os.path.dirname(__file__), "../../Python/specialize.c") - with open(spec_path) as spec_src: - defines = parse_kinds(spec_src) - return defines + def join_row(self, key: str, row_a: tuple, row_b: tuple) -> tuple: + return (key, row_a[0], row_b[0], DiffRatio(row_a[0], row_b[0])) + def join_columns(self, columns: Columns) -> Columns: + return ("Name", "Base Count:", "Head Count:", "Change") -def emit_specialization_stats(opcode_stats, defines): - with Section("Specialization stats", summary="specialization stats by family"): + sort_joined_rows = staticmethod(sort_by_last_column) + + def calculate_rows(self, stats: Stats) -> Rows: + opcode_stats = stats.get_opcode_stats(self.prefix) + total = 0 + counts = [] for name, opcode_stat in opcode_stats.items(): - print_specialization_stats(name, opcode_stat, defines) + if "execution_count" in opcode_stat: + count = opcode_stat["execution_count"] + total += count + miss = 0 + if "specializable" not in opcode_stat: + miss = opcode_stat.get("specialization.miss", 0) + counts.append((count, name, miss)) + counts.sort(reverse=True) + cumulative = 0 + rows: Rows = [] + for count, name, miss in counts: + cumulative += count + if miss: + miss_val = Ratio(miss, count) + else: + miss_val = None + rows.append( + ( + name, + Count(count), + Ratio(count, total), + Ratio(cumulative, total), + miss_val, + ) + ) + return rows -def emit_comparative_specialization_stats( - base_opcode_stats, head_opcode_stats, defines -): - with Section("Specialization stats", summary="specialization stats by family"): - opcodes = set(base_opcode_stats.keys()) & set(head_opcode_stats.keys()) - for opcode in opcodes: - print_comparative_specialization_stats( - opcode, base_opcode_stats[opcode], head_opcode_stats[opcode], defines +class PairCountTable(Table): + columns = ("Pair", "Count:", "Self:", "Cumulative:") + + def calculate_rows(self, stats: Stats) -> Rows: + pair_counts = stats.get_pair_counts("opcode") + total = stats.get_total("opcode") + + cumulative = 0 + rows: Rows = [] + for count, (name_i, name_j) in itertools.islice(pair_counts, 100): + cumulative += count + rows.append( + ( + f"{name_i} {name_j}", + Count(count), + Ratio(count, total), + Ratio(cumulative, total), + ) ) + return rows -def calculate_specialization_effectiveness( - opcode_stats, total, specialized_instructions -): - basic, not_specialized, specialized = categorized_counts( - opcode_stats, specialized_instructions + +def iter_pre_succ_pairs_tables(base_stats: Stats, head_stats: Stats | None = None): + assert head_stats is None + + opcode_stats = base_stats.get_opcode_stats("opcode") + pair_counts = base_stats.get_pair_counts("opcode") + + predecessors: dict[str, collections.Counter] = collections.defaultdict( + collections.Counter + ) + successors: dict[str, collections.Counter] = collections.defaultdict( + collections.Counter ) - return [ - ("Basic", basic, format_ratio(basic, total)), - ("Not specialized", not_specialized, format_ratio(not_specialized, total)), - ("Specialized", specialized, format_ratio(specialized, total)), - ] + total_predecessors: collections.Counter = collections.Counter() + total_successors: collections.Counter = collections.Counter() + for count, (first, second) in pair_counts: + if count: + predecessors[second][first] = count + successors[first][second] = count + total_predecessors[second] += count + total_successors[first] += count + + for name in opcode_stats.keys(): + total1 = total_predecessors[name] + total2 = total_successors[name] + if total1 == 0 and total2 == 0: + continue + pred_rows: Rows = [] + succ_rows: Rows = [] + if total1: + pred_rows = [ + (pred, Count(count), Ratio(count, total1)) + for (pred, count) in predecessors[name].most_common(5) + ] + if total2: + succ_rows = [ + (succ, Count(count), Ratio(count, total2)) + for (succ, count) in successors[name].most_common(5) + ] + + yield Section( + name, + f"Successors and predecessors for {name}", + [ + FixedTable(("Predecessors", "Count:", "Percentage:"), pred_rows), + FixedTable(("Successors", "Count:", "Percentage:"), succ_rows), + ], + ) + + +def iter_specialization_tables(base_stats: Stats, head_stats: Stats | None = None): + class SpecializationTable(SimpleChangeTable): + columns = ("Kind", "Count:", "Ratio:") + def __init__(self, name: str): + self.name = name -def emit_specialization_overview(opcode_stats, total, specialized_instructions): - with Section("Specialization effectiveness"): - rows = calculate_specialization_effectiveness( - opcode_stats, total, specialized_instructions + sort_joined_rows = staticmethod(dont_sort) + + def calculate_rows(self, stats: Stats) -> Rows: + opcode_stats = stats.get_opcode_stats("opcode") + family_stats = opcode_stats[self.name] + total = sum(family_stats.get(kind, 0) for kind in TOTAL) + if total == 0: + return [] + rows: Rows = [] + for key in sorted(family_stats): + if key.startswith("specialization.failure_kinds"): + continue + elif key in ("specialization.hit", "specialization.miss"): + label = key[len("specialization.") :] + elif key in ( + "execution_count", + "specialization.success", + "specialization.failure", + "specializable", + ) or key.startswith("pair"): + continue + else: + label = key + rows.append( + ( + f"{label:>12}", + Count(family_stats[key]), + Ratio(family_stats[key], total), + ) + ) + return rows + + class SpecializationSuccessFailureTable(SimpleChangeTable): + columns = ("", "Count:", "Ratio:") + + def __init__(self, name: str): + self.name = name + + sort_joined_rows = staticmethod(dont_sort) + + def calculate_rows(self, stats: Stats) -> Rows: + opcode_stats = stats.get_opcode_stats("opcode") + family_stats = opcode_stats[self.name] + total_attempts = 0 + for key in ("specialization.success", "specialization.failure"): + total_attempts += family_stats.get(key, 0) + rows: Rows = [] + if total_attempts: + for key in ("specialization.success", "specialization.failure"): + label = key[len("specialization.") :] + label = label[0].upper() + label[1:] + val = family_stats.get(key, 0) + rows.append((label, Count(val), Ratio(val, total_attempts))) + return rows + + class SpecializationFailureKindTable(SimpleChangeTable): + columns = ("Failure kind", "Count:", "Ratio:") + + def __init__(self, name: str): + self.name = name + + def calculate_rows(self, stats: Stats) -> Rows: + def kind_to_text(kind: int, defines: Defines, opname: str): + if kind <= 8: + return pretty(defines[kind][0]) + if opname == "LOAD_SUPER_ATTR": + opname = "SUPER" + elif opname.endswith("ATTR"): + opname = "ATTR" + elif opname in ("FOR_ITER", "SEND"): + opname = "ITER" + elif opname.endswith("SUBSCR"): + opname = "SUBSCR" + for name in defines[kind]: + if name.startswith(opname): + return pretty(name[len(opname) + 1 :]) + return "kind " + str(kind) + + defines = stats.defines + opcode_stats = stats.get_opcode_stats("opcode") + family_stats = opcode_stats[self.name] + total_failures = family_stats.get("specialization.failure", 0) + failure_kinds = [0] * 40 + for key in family_stats: + if not key.startswith("specialization.failure_kind"): + continue + index = int(key[:-1].split("[")[1]) + failure_kinds[index] = family_stats[key] + failures = [(value, index) for (index, value) in enumerate(failure_kinds)] + failures.sort(reverse=True) + rows: Rows = [] + for value, index in failures: + if not value: + continue + rows.append( + ( + kind_to_text(index, defines, self.name), + Count(value), + Ratio(value, total_failures), + ) + ) + return rows + + opcode_base_stats = base_stats.get_opcode_stats("opcode") + names = opcode_base_stats.keys() + if head_stats is not None: + opcode_head_stats = head_stats.get_opcode_stats("opcode") + names &= opcode_head_stats.keys() # type: ignore + else: + opcode_head_stats = {} + + for name in sorted(names): + if "specializable" not in opcode_base_stats.get(name, {}): + continue + total = sum( + stats.get(name, {}).get(kind, 0) + for kind in TOTAL + for stats in (opcode_base_stats, opcode_head_stats) ) - emit_table(("Instructions", "Count:", "Ratio:"), rows) - for title, field in ( - ("Deferred", "specialization.deferred"), - ("Misses", "specialization.miss"), - ): + if total == 0: + continue + yield Section( + name, + f"specialization stats for {name} family", + [ + SpecializationTable(name), + SpecializationSuccessFailureTable(name), + SpecializationFailureKindTable(name), + ], + ) + + +def iter_specialization_effectiveness_tables( + base_stats: Stats, head_stats: Stats | None = None +): + class SpecializationEffectivenessTable(SimpleChangeTable): + columns = ("Instructions", "Count:", "Ratio:") + + sort_joined_rows = staticmethod(dont_sort) + + def calculate_rows(self, stats: Stats) -> Rows: + opcode_stats = stats.get_opcode_stats("opcode") + total = stats.get_total("opcode") + specialized_instructions = stats.specialized_instructions + + basic = 0 + specialized = 0 + not_specialized = 0 + for name, opcode_stat in opcode_stats.items(): + if "execution_count" not in opcode_stat: + continue + count = opcode_stat["execution_count"] + if "specializable" in opcode_stat: + not_specialized += count + elif name in specialized_instructions: + miss = opcode_stat.get("specialization.miss", 0) + not_specialized += miss + specialized += count - miss + else: + basic += count + + return [ + ("Basic", Count(basic), Ratio(basic, total)), + ( + "Not specialized", + Count(not_specialized), + Ratio(not_specialized, total), + ), + ("Specialized", Count(specialized), Ratio(specialized, total)), + ] + + class DeferredByInstructionTable(SimpleChangeTable): + columns = ("Name", "Count:", "Ratio:") + + def calculate_rows(self, stats: Stats) -> Rows: + opcode_stats = stats.get_opcode_stats("opcode") + total = 0 counts = [] for name, opcode_stat in opcode_stats.items(): - # Avoid double counting misses - if title == "Misses" and "specializable" in opcode_stat: - continue - value = opcode_stat.get(field, 0) + value = opcode_stat.get("specialization.deferred", 0) counts.append((value, name)) total += value counts.sort(reverse=True) if total: - with Section(f"{title} by instruction", 3): - rows = [ - (name, count, format_ratio(count, total)) - for (count, name) in counts[:10] - ] - emit_table(("Name", "Count:", "Ratio:"), rows) - - -def emit_comparative_specialization_overview( - base_opcode_stats, - base_total, - head_opcode_stats, - head_total, - specialized_instructions, -): - with Section("Specialization effectiveness"): - base_rows = calculate_specialization_effectiveness( - base_opcode_stats, base_total, specialized_instructions - ) - head_rows = calculate_specialization_effectiveness( - head_opcode_stats, head_total, specialized_instructions - ) - emit_table( - ( - "Instructions", - "Base Count:", - "Base Ratio:", - "Head Count:", - "Head Ratio:", - ), - join_rows(base_rows, head_rows), - ) - + return [ + (name, Count(count), Ratio(count, total)) + for (count, name) in counts[:10] + ] + else: + return [] -def get_stats_defines(): - stats_path = os.path.join( - os.path.dirname(__file__), "../../Include/cpython/pystats.h" - ) - with open(stats_path) as stats_src: - defines = parse_kinds(stats_src, prefix="EVAL_CALL") - return defines - - -def calculate_call_stats(stats, defines): - total = 0 - for key, value in stats.items(): - if "Calls to" in key: - total += value - rows = [] - for key, value in stats.items(): - if "Calls to" in key: - rows.append((key, value, format_ratio(value, total))) - elif key.startswith("Calls "): - name, index = key[:-1].split("[") - index = int(index) - label = name + " (" + pretty(defines[index][0]) + ")" - rows.append((label, value, format_ratio(value, total))) - for key, value in stats.items(): - if key.startswith("Frame"): - rows.append((key, value, format_ratio(value, total))) - return rows - - -def emit_call_stats(stats, defines): - with Section("Call stats", summary="Inlined calls and frame stats"): - rows = calculate_call_stats(stats, defines) - emit_table(("", "Count:", "Ratio:"), rows) - - -def emit_comparative_call_stats(base_stats, head_stats, defines): - with Section("Call stats", summary="Inlined calls and frame stats"): - base_rows = calculate_call_stats(base_stats, defines) - head_rows = calculate_call_stats(head_stats, defines) - rows = join_rows(base_rows, head_rows) - rows.sort(key=lambda x: -percentage_to_float(x[-1])) - emit_table( - ("", "Base Count:", "Base Ratio:", "Head Count:", "Head Ratio:"), rows - ) + class MissesByInstructionTable(SimpleChangeTable): + columns = ("Name", "Count:", "Ratio:") + def calculate_rows(self, stats: Stats) -> Rows: + opcode_stats = stats.get_opcode_stats("opcode") -def calculate_object_stats(stats): - total_materializations = stats.get("Object new values") - total_allocations = stats.get("Object allocations") + stats.get( - "Object allocations from freelist" - ) - total_increfs = stats.get("Object interpreter increfs") + stats.get( - "Object increfs" - ) - total_decrefs = stats.get("Object interpreter decrefs") + stats.get( - "Object decrefs" - ) - rows = [] - for key, value in stats.items(): - if key.startswith("Object"): - if "materialize" in key: - ratio = format_ratio(value, total_materializations) - elif "allocations" in key: - ratio = format_ratio(value, total_allocations) - elif "increfs" in key: - ratio = format_ratio(value, total_increfs) - elif "decrefs" in key: - ratio = format_ratio(value, total_decrefs) + total = 0 + counts = [] + for name, opcode_stat in opcode_stats.items(): + # Avoid double counting misses + if "specializable" in opcode_stat: + continue + value = opcode_stat.get("specialization.misses", 0) + counts.append((value, name)) + total += value + counts.sort(reverse=True) + if total: + return [ + (name, Count(count), Ratio(count, total)) + for (count, name) in counts[:10] + ] else: - ratio = "" - label = key[6:].strip() - label = label[0].upper() + label[1:] - rows.append((label, value, ratio)) - return rows - - -def calculate_gc_stats(stats): - gc_stats = [] - for key, value in stats.items(): - if not key.startswith("GC"): - continue - n, _, rest = key[3:].partition("]") - name = rest.strip() - gen_n = int(n) - while len(gc_stats) <= gen_n: - gc_stats.append({}) - gc_stats[gen_n][name] = value - return [ - (i, gen["collections"], gen["objects collected"], gen["object visits"]) - for (i, gen) in enumerate(gc_stats) - ] - - -def emit_object_stats(stats): - with Section("Object stats", summary="allocations, frees and dict materializatons"): - rows = calculate_object_stats(stats) - emit_table(("", "Count:", "Ratio:"), rows) - - -def emit_comparative_object_stats(base_stats, head_stats): - with Section("Object stats", summary="allocations, frees and dict materializatons"): - base_rows = calculate_object_stats(base_stats) - head_rows = calculate_object_stats(head_stats) - emit_table( - ("", "Base Count:", "Base Ratio:", "Head Count:", "Head Ratio:"), - join_rows(base_rows, head_rows), - ) + return [] + yield SpecializationEffectivenessTable() + yield Section("Deferred by instruction", "", [DeferredByInstructionTable()]) + yield Section("Misses by instruction", "", [MissesByInstructionTable()]) -def emit_gc_stats(stats): - with Section("GC stats", summary="GC collections and effectiveness"): - rows = calculate_gc_stats(stats) - emit_table( - ("Generation:", "Collections:", "Objects collected:", "Object visits:"), - rows, - ) +class CallStatsTable(SimpleChangeTable): + columns = ("", "Count:", "Ratio:") -def emit_comparative_gc_stats(base_stats, head_stats): - with Section("GC stats", summary="GC collections and effectiveness"): - base_rows = calculate_gc_stats(base_stats) - head_rows = calculate_gc_stats(head_stats) - emit_table( - ( - "Generation:", - "Base collections:", - "Head collections:", - "Base objects collected:", - "Head objects collected:", - "Base object visits:", - "Head object visits:", - ), - join_rows(base_rows, head_rows), - ) + sort_joined_rows = staticmethod(dont_sort) + def calculate_rows(self, stats: Stats) -> Rows: + defines = stats.pystats_defines -def get_total(opcode_stats): - total = 0 - for opcode_stat in opcode_stats.values(): - if "execution_count" in opcode_stat: - total += opcode_stat["execution_count"] - return total + total = 0 + for key, value in stats.items(): + if "Calls to" in key: + total += value + rows: Rows = [] + for key, value in sorted(stats.items()): + if "Calls to" in key: + rows.append((key, Count(value), Ratio(value, total))) + elif key.startswith("Calls "): + name, index = key[:-1].split("[") + index = int(index) + label = f"{name} ({pretty(defines[index][0])})" + rows.append((label, Count(value), Ratio(value, total))) -def emit_pair_counts(opcode_stats, total): - pair_counts = [] - for name_i, opcode_stat in opcode_stats.items(): - for key, value in opcode_stat.items(): - if key.startswith("pair_count"): - name_j, _, _ = key[11:].partition("]") - if value: - pair_counts.append((value, (name_i, name_j))) - with Section("Pair counts", summary="Pair counts for top 100 pairs"): - pair_counts.sort(reverse=True) - cumulative = 0 - rows = [] - for count, pair in itertools.islice(pair_counts, 100): - name_i, name_j = pair - cumulative += count - rows.append( - ( - f"{name_i} {name_j}", - count, - format_ratio(count, total), - format_ratio(cumulative, total), - ) - ) - emit_table(("Pair", "Count:", "Self:", "Cumulative:"), rows) - with Section( - "Predecessor/Successor Pairs", - summary="Top 5 predecessors and successors of each opcode", - ): - predecessors = collections.defaultdict(collections.Counter) - successors = collections.defaultdict(collections.Counter) - total_predecessors = collections.Counter() - total_successors = collections.Counter() - for count, (first, second) in pair_counts: - if count: - predecessors[second][first] = count - successors[first][second] = count - total_predecessors[second] += count - total_successors[first] += count - for name in opcode_stats.keys(): - total1 = total_predecessors[name] - total2 = total_successors[name] - if total1 == 0 and total2 == 0: - continue - pred_rows = succ_rows = () - if total1: - pred_rows = [ - (pred, count, f"{count/total1:.1%}") - for (pred, count) in predecessors[name].most_common(5) - ] - if total2: - succ_rows = [ - (succ, count, f"{count/total2:.1%}") - for (succ, count) in successors[name].most_common(5) - ] - with Section(name, 3, f"Successors and predecessors for {name}"): - emit_table(("Predecessors", "Count:", "Percentage:"), pred_rows) - emit_table(("Successors", "Count:", "Percentage:"), succ_rows) - - -def calculate_optimization_stats(stats): - attempts = stats["Optimization attempts"] - created = stats["Optimization traces created"] - executed = stats["Optimization traces executed"] - uops = stats["Optimization uops executed"] - trace_stack_overflow = stats["Optimization trace stack overflow"] - trace_stack_underflow = stats["Optimization trace stack underflow"] - trace_too_long = stats["Optimization trace too long"] - trace_too_short = stats["Optimiztion trace too short"] - inner_loop = stats["Optimization inner loop"] - recursive_call = stats["Optimization recursive call"] - - return [ - ("Optimization attempts", attempts, ""), - ("Traces created", created, format_ratio(created, attempts)), - ("Traces executed", executed, ""), - ("Uops executed", uops, int(uops / (executed or 1))), - ("Trace stack overflow", trace_stack_overflow, ""), - ("Trace stack underflow", trace_stack_underflow, ""), - ("Trace too long", trace_too_long, ""), - ("Trace too short", trace_too_short, ""), - ("Inner loop found", inner_loop, ""), - ("Recursive call", recursive_call, ""), - ] - - -def calculate_uop_execution_counts(opcode_stats): - total = 0 - counts = [] - for name, opcode_stat in opcode_stats.items(): - if "execution_count" in opcode_stat: - count = opcode_stat["execution_count"] - counts.append((count, name)) - total += count - counts.sort(reverse=True) - cumulative = 0 - rows = [] - for count, name in counts: - cumulative += count - rows.append( - (name, count, format_ratio(count, total), format_ratio(cumulative, total)) - ) - return rows + for key, value in sorted(stats.items()): + if key.startswith("Frame"): + rows.append((key, Count(value), Ratio(value, total))) + return rows -def emit_optimization_stats(stats): - if "Optimization attempts" not in stats: - return - uop_stats = extract_opcode_stats(stats, "uops") +class ObjectStatsTable(SimpleChangeTable): + columns = ("", "Count:", "Ratio:") - with Section( - "Optimization (Tier 2) stats", summary="statistics about the Tier 2 optimizer" - ): - with Section("Overall stats", level=3): - rows = calculate_optimization_stats(stats) - emit_table(("", "Count:", "Ratio:"), rows) - - emit_histogram( - "Trace length histogram", - stats, - "Trace length", - stats["Optimization traces created"], + def calculate_rows(self, stats: Stats) -> Rows: + total_materializations = stats.get("Object new values", 0) + total_allocations = stats.get("Object allocations", 0) + stats.get( + "Object allocations from freelist", 0 ) - emit_histogram( - "Optimized trace length histogram", - stats, - "Optimized trace length", - stats["Optimization traces created"], + total_increfs = stats.get("Object interpreter increfs", 0) + stats.get( + "Object increfs", 0 ) - emit_histogram( - "Trace run length histogram", - stats, - "Trace run length", - stats["Optimization traces executed"], + total_decrefs = stats.get("Object interpreter decrefs", 0) + stats.get( + "Object decrefs", 0 ) + rows: Rows = [] + for key, value in stats.items(): + if key.startswith("Object"): + if "materialize" in key: + ratio = Ratio(value, total_materializations) + elif "allocations" in key: + ratio = Ratio(value, total_allocations) + elif "increfs" in key: + ratio = Ratio(value, total_increfs) + elif "decrefs" in key: + ratio = Ratio(value, total_decrefs) + else: + ratio = None + label = key[6:].strip() + label = label[0].upper() + label[1:] + rows.append((label, Count(value), ratio)) + return rows + + +class GCStatsTable(Table): + columns = ("Generation:", "Collections:", "Objects collected:", "Object visits:") + + def calculate_rows(self, stats: Stats) -> Rows: + gc_stats: list[dict[str, int]] = [] + for key, value in stats.items(): + if not key.startswith("GC"): + continue + n, _, rest = key[3:].partition("]") + name = rest.strip() + gen_n = int(n) + while len(gc_stats) <= gen_n: + gc_stats.append({}) + gc_stats[gen_n][name] = value + return [ + ( + Count(i), + Count(gen["collections"]), + Count(gen["objects collected"]), + Count(gen["object visits"]), + ) + for (i, gen) in enumerate(gc_stats) + ] - with Section("Uop stats", level=3): - rows = calculate_uop_execution_counts(uop_stats) - emit_table(("Uop", "Count:", "Self:", "Cumulative:"), rows) - with Section("Unsupported opcodes", level=3): - unsupported_opcodes = extract_opcode_stats(stats, "unsupported_opcode") +def iter_optimization_tables(base_stats: Stats, head_stats: Stats | None = None): + class OptimizationStatsTable(SimpleChangeTable): + columns = ("", "Count:", "Ratio:") + + sort_joined_rows = staticmethod(dont_sort) + + def calculate_rows(self, stats: Stats) -> Rows: + if "Optimization attempts" not in stats: + return [] + + attempts = stats["Optimization attempts"] + created = stats["Optimization traces created"] + executed = stats["Optimization traces executed"] + uops = stats["Optimization uops executed"] + trace_stack_overflow = stats["Optimization trace stack overflow"] + trace_stack_underflow = stats["Optimization trace stack underflow"] + trace_too_long = stats["Optimization trace too long"] + trace_too_short = stats["Optimization trace too short"] + inner_loop = stats["Optimization inner loop"] + recursive_call = stats["Optimization recursive call"] + + return [ + ("Optimization attempts", Count(attempts), ""), + ("Traces created", Count(created), Ratio(created, attempts)), + ("Traces executed", Count(executed), ""), + ("Uops executed", Count(uops), CountPer(uops, executed)), + ( + "Trace stack overflow", + Count(trace_stack_overflow), + Ratio(trace_stack_overflow, created), + ), + ( + "Trace stack underflow", + Count(trace_stack_underflow), + Ratio(trace_stack_underflow, created), + ), + ( + "Trace too long", + Count(trace_too_long), + Ratio(trace_too_long, created), + ), + ( + "Trace too short", + Count(trace_too_short), + Ratio(trace_too_short, created), + )("Inner loop found", Count(inner_loop), Ratio(inner_loop, created)), + ( + "Recursive call", + Count(recursive_call), + Ratio(recursive_call, created), + ), + ] + + class HistogramTable(SimpleChangeTable): + columns = ("Range", "Count:", "Ratio:") + + def __init__(self, key: str, den: str): + self.key = key + self.den = den + + sort_joined_rows = staticmethod(dont_sort) + + def calculate_rows(self, stats: Stats) -> Rows: + rows: Rows = [] + for k, v in stats.items(): + if k.startswith(self.key): + match = re.match(r".+\[([0-9]+)\]", k) + if match is not None: + entry = int(match.groups()[0]) + rows.append( + (f"<= {entry}", Count(v), Ratio(int(v), stats[self.den])) + ) + return rows + + class UnsupportedOpcodesTable(SimpleChangeTable): + columns = ("Opcode", "Count:") + + def calculate_rows(self, stats: Stats) -> Rows: + unsupported_opcodes = stats.get_opcode_stats("unsupported_opcode") data = [] for opcode, entry in unsupported_opcodes.items(): - data.append((entry["count"], opcode)) + data.append((Count(entry["count"]), opcode)) data.sort(reverse=True) - rows = [(x[1], x[0]) for x in data] - emit_table(("Opcode", "Count"), rows) + return [(x[1], x[0]) for x in data] + if "Optimization attempts" not in base_stats or ( + head_stats is not None and "Optimization attempts" not in head_stats + ): + return -def emit_comparative_optimization_stats(base_stats, head_stats): - print("## Comparative optimization stats not implemented\n\n") + yield OptimizationStatsTable() + for name, den in [ + ("Trace length", "Optimization traces created"), + ("Optimized trace length", "Optimization traces created"), + ("Trace run length", "Optimization traces executed"), + ]: + yield Section(f"{name} histogram", "", [HistogramTable(name, den)]) -def output_single_stats(stats): - opcode_stats = extract_opcode_stats(stats, "opcode") - total = get_total(opcode_stats) - emit_execution_counts(opcode_stats, total) - emit_pair_counts(opcode_stats, total) - emit_specialization_stats(opcode_stats, stats["_defines"]) - emit_specialization_overview( - opcode_stats, total, stats["_specialized_instructions"] - ) - emit_call_stats(stats, stats["_stats_defines"]) - emit_object_stats(stats) - emit_gc_stats(stats) - emit_optimization_stats(stats) - with Section("Meta stats", summary="Meta statistics"): - emit_table(("", "Count:"), [("Number of data files", stats["__nfiles__"])]) + yield Section("Uop stats", "", [ExecutionCountTable("uops")]) + yield Section("Unsupported opcodes", "", [UnsupportedOpcodesTable()]) -def output_comparative_stats(base_stats, head_stats): - base_opcode_stats = extract_opcode_stats(base_stats, "opcode") - base_total = get_total(base_opcode_stats) +class MetaStatsTable(Table): + columns = ("", "Count:") - head_opcode_stats = extract_opcode_stats(head_stats, "opcode") - head_total = get_total(head_opcode_stats) + def calculate_rows(self, stats: Stats) -> Rows: + return [("Number of data files", Count(stats.get("__nfiles__", 0)))] - emit_comparative_execution_counts( - base_opcode_stats, base_total, head_opcode_stats, head_total - ) - emit_comparative_specialization_stats( - base_opcode_stats, head_opcode_stats, head_stats["_defines"] - ) - emit_comparative_specialization_overview( - base_opcode_stats, - base_total, - head_opcode_stats, - head_total, - head_stats["_specialized_instructions"], - ) - emit_comparative_call_stats(base_stats, head_stats, head_stats["_stats_defines"]) - emit_comparative_object_stats(base_stats, head_stats) - emit_comparative_gc_stats(base_stats, head_stats) - emit_comparative_optimization_stats(base_stats, head_stats) +class Section: + def __init__( + self, + title: str, + summary: str = "", + part_iter=None, + comparative: bool = True, + ): + self.title = title + if not summary: + self.summary = title.lower() + else: + self.summary = summary + if part_iter is None: + part_iter = [] + if isinstance(part_iter, list): + + def iter_parts(base_stats: Stats, head_stats: Stats | None): + yield from part_iter -def output_stats(inputs, json_output=None): + self.part_iter = iter_parts + else: + self.part_iter = part_iter + self.comparative = comparative + + def output_markdown( + self, + out: TextIO, + base_stats: Stats, + head_stats: Stats | None = None, + level: int = 1, + ) -> None: + if self.title: + print("#" * level, self.title, file=out) + print(file=out) + print("
", file=out) + print("", self.summary, "", file=out) + print(file=out) + if head_stats is not None and self.comparative is False: + print("Not included in comparative output.\n") + else: + for part in self.part_iter(base_stats, head_stats): + part.output_markdown(out, base_stats, head_stats, level=level + 1) + print(file=out) + if self.title: + print("
", file=out) + print(file=out) + + +LAYOUT = Section( + "", + "", + [ + Section( + "Execution counts", + "execution counts for all instructions", + [ExecutionCountTable("opcode")], + ), + Section( + "Pair counts", + "Pair counts for top 100 pairs", + [PairCountTable()], + comparative=False, + ), + Section( + "Predecessor/Successor Pairs", + "Top 5 predecessors and successors of each opcode", + iter_pre_succ_pairs_tables, + comparative=False, + ), + Section( + "Specialization stats", + "specialization stats by family", + iter_specialization_tables, + ), + Section( + "Specialization effectiveness", + "", + iter_specialization_effectiveness_tables, + ), + Section("Call stats", "Inlined calls and frame stats", [CallStatsTable()]), + Section( + "Object stats", + "allocations, frees and dict materializatons", + [ObjectStatsTable()], + ), + Section("GC stats", "GC collections and effectiveness", [GCStatsTable()]), + Section( + "Optimization (Tier 2) stats", + "statistics about the Tier 2 optimizer", + iter_optimization_tables, + ), + Section("Meta stats", "Meta statistics", [MetaStatsTable()]), + ], +) + + +def output_markdown(out: TextIO, base_stats: Stats, head_stats: Stats | None = None): + LAYOUT.output_markdown(out, base_stats, head_stats) + print("---", file=out) + print("Stats gathered on:", date.today(), file=out) + + +def output_stats(inputs: list[Path], json_output=None): if len(inputs) == 1: - stats = gather_stats(inputs[0]) + stats = Stats(Path(inputs[0])) if json_output is not None: json.dump(stats, json_output) - output_single_stats(stats) + output_markdown(sys.stdout, stats) elif len(inputs) == 2: if json_output is not None: raise ValueError("Can not output to JSON when there are multiple inputs") - base_stats = gather_stats(inputs[0]) - head_stats = gather_stats(inputs[1]) - output_comparative_stats(base_stats, head_stats) - - print("---") - print("Stats gathered on:", date.today()) + base_stats = Stats(Path(inputs[0])) + head_stats = Stats(Path(inputs[1])) + output_markdown(sys.stdout, base_stats, head_stats) def main(): From 4925c33a0e414543e0a5862474911d30c2effbbf Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Thu, 5 Oct 2023 09:58:00 -0400 Subject: [PATCH 2/8] Truncate histograms --- Tools/scripts/summarize_stats.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index 103d3be6a7c86b..2f966f6a1046da 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -822,14 +822,19 @@ def __init__(self, key: str, den: str): def calculate_rows(self, stats: Stats) -> Rows: rows: Rows = [] + last_non_zero = 0 for k, v in stats.items(): if k.startswith(self.key): match = re.match(r".+\[([0-9]+)\]", k) if match is not None: entry = int(match.groups()[0]) + if v != 0: + last_non_zero = len(rows) rows.append( (f"<= {entry}", Count(v), Ratio(int(v), stats[self.den])) ) + # Don't include any zero entries at the end + rows = rows[:last_non_zero + 1] return rows class UnsupportedOpcodesTable(SimpleChangeTable): From 901b95278d9804a173009038f6d7600448c7c5e0 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Thu, 5 Oct 2023 11:10:56 -0400 Subject: [PATCH 3/8] Include commas in histogram bins --- Tools/scripts/summarize_stats.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index 2f966f6a1046da..5550a1c022c068 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -831,10 +831,10 @@ def calculate_rows(self, stats: Stats) -> Rows: if v != 0: last_non_zero = len(rows) rows.append( - (f"<= {entry}", Count(v), Ratio(int(v), stats[self.den])) + (f"<= {entry:,d}", Count(v), Ratio(int(v), stats[self.den])) ) # Don't include any zero entries at the end - rows = rows[:last_non_zero + 1] + rows = rows[: last_non_zero + 1] return rows class UnsupportedOpcodesTable(SimpleChangeTable): From fa176e71585a68b5e36064c5a6cd68e3a813df79 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Thu, 5 Oct 2023 11:42:53 -0400 Subject: [PATCH 4/8] Don't have stats be a subclass of dict --- Tools/scripts/summarize_stats.py | 54 ++++++++++++++++++++++---------- 1 file changed, 38 insertions(+), 16 deletions(-) diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index 5550a1c022c068..4d227ba3baec4a 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -43,21 +43,22 @@ def pretty(name: str) -> str: return name.replace("_", " ").lower() -class Stats(dict): +class Stats: def __init__(self, input: Path): super().__init__() if input.is_file(): with open(input, "r") as fd: - self.update(json.load(fd)) + self._data = json.load(fd) - self["_stats_defines"] = { + self._data["_stats_defines"] = { int(k): v for k, v in self["_stats_defines"].items() } - self["_defines"] = {int(k): v for k, v in self["_defines"].items()} + self._data["_defines"] = {int(k): v for k, v in self["_defines"].items()} elif input.is_dir(): - stats: collections.Counter = collections.Counter() + stats = collections.Counter() + for filename in input.iterdir(): with open(filename) as fd: for line in fd: @@ -72,8 +73,7 @@ def __init__(self, input: Path): stats[key.strip()] += int(value) stats["__nfiles__"] += 1 - self.update(stats) - + self._data = dict(stats) self._load_metadata_from_source() else: @@ -81,6 +81,27 @@ def __init__(self, input: Path): self._opcode_stats: dict[str, OpcodeStats] = {} + def __getitem__(self, key): + return self._data[key] + + def __contains__(self, key): + return key in self._data + + def get(self, key, default=None): + return self._data.get(key, default) + + def items(self): + return self._data.items() + + def keys(self): + return self._data.keys() + + def values(self): + return self._data.values() + + def save(self, json_output: TextIO): + json.dump(self._data, json_output) + def _load_metadata_from_source(self): def get_defines(filepath: Path, prefix: str = "SPEC_FAIL") -> Defines: with open(SOURCE_DIR / filepath) as spec_src: @@ -97,27 +118,27 @@ def get_defines(filepath: Path, prefix: str = "SPEC_FAIL") -> Defines: import opcode - self["_specialized_instructions"] = [ + self._data["_specialized_instructions"] = [ op for op in opcode._specialized_opmap.keys() # type: ignore if "__" not in op ] - self["_stats_defines"] = get_defines( + self._data["_stats_defines"] = get_defines( Path("Include") / "cpython" / "pystats.h", "EVAL_CALL" ) - self["_defines"] = get_defines(Path("Python") / "specialize.c") + self._data["_defines"] = get_defines(Path("Python") / "specialize.c") @property def defines(self) -> Defines: - return self["_defines"] + return self._data["_defines"] @property def pystats_defines(self) -> Defines: - return self["_stats_defines"] + return self._data["_stats_defines"] @property def specialized_instructions(self) -> list[str]: - return self["_specialized_instructions"] + return self._data["_specialized_instructions"] def get_opcode_stats(self, prefix: str) -> OpcodeStats: if prefix in self._opcode_stats: @@ -803,7 +824,8 @@ def calculate_rows(self, stats: Stats) -> Rows: "Trace too short", Count(trace_too_short), Ratio(trace_too_short, created), - )("Inner loop found", Count(inner_loop), Ratio(inner_loop, created)), + ), + ("Inner loop found", Count(inner_loop), Ratio(inner_loop, created)), ( "Recursive call", Count(recursive_call), @@ -976,11 +998,11 @@ def output_markdown(out: TextIO, base_stats: Stats, head_stats: Stats | None = N print("Stats gathered on:", date.today(), file=out) -def output_stats(inputs: list[Path], json_output=None): +def output_stats(inputs: list[Path], json_output=TextIO | None): if len(inputs) == 1: stats = Stats(Path(inputs[0])) if json_output is not None: - json.dump(stats, json_output) + stats.save(json_output) output_markdown(sys.stdout, stats) elif len(inputs) == 2: if json_output is not None: From 193b4503d1f94151da65ecfe5852a00c88a25207 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Thu, 5 Oct 2023 16:27:45 -0400 Subject: [PATCH 5/8] Use a more functional approach --- Tools/scripts/summarize_stats.py | 1074 ++++++++++++++++-------------- 1 file changed, 578 insertions(+), 496 deletions(-) diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index 4d227ba3baec4a..ed38ff201e7924 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -11,13 +11,14 @@ import argparse import collections from datetime import date +import enum import itertools import json import os from pathlib import Path import re import sys -from typing import TextIO, TypeAlias +from typing import Callable, TextIO, TypeAlias OpcodeStats: TypeAlias = dict[str, dict[str, int]] @@ -25,6 +26,7 @@ Defines: TypeAlias = dict[int, list[str]] Rows: TypeAlias = list[tuple] Columns: TypeAlias = tuple[str, ...] +RowCalculator: TypeAlias = Callable[["Stats"], Rows] if os.name == "nt": @@ -44,6 +46,8 @@ def pretty(name: str) -> str: class Stats: + _data: dict + def __init__(self, input: Path): super().__init__() @@ -57,7 +61,7 @@ def __init__(self, input: Path): self._data["_defines"] = {int(k): v for k, v in self["_defines"].items()} elif input.is_dir(): - stats = collections.Counter() + stats = collections.Counter[str]() for filename in input.iterdir(): with open(filename) as fd: @@ -177,25 +181,14 @@ def markdown(self) -> str: return format(self, ",d") -class CountPer: - def __init__(self, num: int, den: int): +class Ratio: + def __init__(self, num: int, den: int, percentage: bool = True): self.num = num self.den = den + self.percentage = percentage if den == 0 and num != 0: raise ValueError("Invalid denominator") - def markdown(self) -> str: - if self.den == 0: - return "0" - else: - return f"{int(self.num / self.den):,d}" - - -class Ratio: - def __init__(self, num: int, den: int): - self.num = num - self.den = den - def __float__(self): if self.den == 0: return 0.0 @@ -205,8 +198,10 @@ def __float__(self): def markdown(self) -> str: if self.den == 0: return "" - else: + elif self.percentage: return f"{self.num / self.den:,.01%}" + else: + return f"{self.num / self.den:,.02f}" class DiffRatio(Ratio): @@ -217,32 +212,63 @@ def __init__(self, base: int | str, head: int | str): super().__init__(head - base, base) -def sort_by_last_column(rows: Rows): - rows.sort(key=lambda row: abs(float(row[-1])), reverse=True) - - -def dont_sort(rows: Rows): - pass +class JoinMode(enum.Enum): + # Join using the first column as a key + SIMPLE = 0 + # Join using the first column as a key, and indicate the change in the + # second column of each input table as a new column + CHANGE = 1 + # Join using the first column as a key, indicating the change in the second + # column of each input table as a ne column, and omit all other columns + CHANGE_ONE_COLUMN = 2 class Table: - columns: Columns - sort_by_last_column: bool = False + """ + A Table defines how to convert a set of Stats into a specific set of rows + displaying some aspect of the data. + """ - def calculate_rows(self, stats: Stats) -> Rows: - raise NotImplementedError() + def __init__( + self, + column_names: Columns, + calc_rows: RowCalculator, + join_mode: JoinMode = JoinMode.SIMPLE, + ): + self.columns = column_names + self.calc_rows = calc_rows + self.join_mode = join_mode def join_row(self, key: str, row_a: tuple, row_b: tuple) -> tuple: - return (key, *row_a, *row_b) + match self.join_mode: + case JoinMode.SIMPLE: + return (key, *row_a, *row_b) + case JoinMode.CHANGE: + return (key, *row_a, *row_b, DiffRatio(row_a[0], row_b[0])) + case JoinMode.CHANGE_ONE_COLUMN: + return (key, row_a[0], row_b[0], DiffRatio(row_a[0], row_b[0])) def join_columns(self, columns: Columns) -> Columns: - return ( - columns[0], - *("Base " + x for x in columns[1:]), - *("Head " + x for x in columns[1:]), - ) - - sort_joined_rows = staticmethod(dont_sort) + match self.join_mode: + case JoinMode.SIMPLE: + return ( + columns[0], + *("Base " + x for x in columns[1:]), + *("Head " + x for x in columns[1:]), + ) + case JoinMode.CHANGE: + return ( + columns[0], + *("Base " + x for x in columns[1:]), + *("Head " + x for x in columns[1:]), + ) + ("Change:",) + case JoinMode.CHANGE_ONE_COLUMN: + return ( + columns[0], + "Base " + columns[1], + "Head " + columns[1], + "Change:", + ) def join_tables(self, rows_a: Rows, rows_b: Rows) -> tuple[Columns, Rows]: ncols = len(self.columns) @@ -261,105 +287,58 @@ def join_tables(self, rows_a: Rows, rows_b: Rows) -> tuple[Columns, Rows]: self.join_row(k, data_a.get(k, default), data_b.get(k, default)) for k in keys ] - if self.sort_by_last_column: + if self.join_mode in (JoinMode.CHANGE, JoinMode.CHANGE_ONE_COLUMN): rows.sort(key=lambda row: abs(float(row[-1])), reverse=True) columns = self.join_columns(self.columns) return columns, rows - def get_rows( + def get_table( self, base_stats: Stats, head_stats: Stats | None = None ) -> tuple[Columns, Rows]: if head_stats is None: - rows = self.calculate_rows(base_stats) + rows = self.calc_rows(base_stats) return self.columns, rows else: - rows_a = self.calculate_rows(base_stats) - rows_b = self.calculate_rows(head_stats) + rows_a = self.calc_rows(base_stats) + rows_b = self.calc_rows(head_stats) cols, rows = self.join_tables(rows_a, rows_b) return cols, rows - def output_markdown( - self, - out: TextIO, - base_stats: Stats, - head_stats: Stats | None = None, - level: int = 2, - ) -> None: - header, rows = self.get_rows(base_stats, head_stats) - if len(rows) == 0: - return - def to_markdown(x): - if hasattr(x, "markdown"): - return x.markdown() - elif isinstance(x, str): - return x - elif x is None: - return "" - else: - raise TypeError(f"Can't convert {x} to markdown") - - width = len(header) - header_line = "|" - under_line = "|" - for item in header: - under = "---" - if item.endswith(":"): - item = item[:-1] - under += ":" - header_line += item + " | " - under_line += under + "|" - print(header_line, file=out) - print(under_line, file=out) - for row in rows: - if width is not None and len(row) != width: - raise ValueError("Wrong number of elements in row '" + str(row) + "'") - print("|", " | ".join(to_markdown(i) for i in row), "|", file=out) - print(file=out) - - -class FixedTable(Table): - def __init__(self, columns: Columns, rows: Rows): - self.columns = columns - self.rows = rows - - def get_rows(self, *args) -> tuple[Columns, Rows]: - return self.columns, self.rows - - -class SimpleChangeTable(Table): +class Section: """ - Base class of tables where the comparison table has an extra column "Change" - computed from the change of the second column of the base and head. Sorted - by the "Change" column. + A Section defines a section of the output document. """ - def join_row(self, key: str, base_data: tuple, head_data: tuple) -> tuple: - return (key, *base_data, *head_data, DiffRatio(base_data[0], head_data[0])) - - def join_columns(self, columns: Columns) -> Columns: - return super().join_columns(columns) + ("Change:",) - - sort_joined_rows = staticmethod(sort_by_last_column) - - -class ExecutionCountTable(Table): - columns = ("Name", "Count:", "Self:", "Cumulative:", "Miss ratio:") - - def __init__(self, prefix: str): - self.prefix = prefix + def __init__( + self, + title: str = "", + summary: str = "", + part_iter=None, + comparative: bool = True, + ): + self.title = title + if not summary: + self.summary = title.lower() + else: + self.summary = summary + if part_iter is None: + part_iter = [] + if isinstance(part_iter, list): - def join_row(self, key: str, row_a: tuple, row_b: tuple) -> tuple: - return (key, row_a[0], row_b[0], DiffRatio(row_a[0], row_b[0])) + def iter_parts(base_stats: Stats, head_stats: Stats | None): + yield from part_iter - def join_columns(self, columns: Columns) -> Columns: - return ("Name", "Base Count:", "Head Count:", "Change") + self.part_iter = iter_parts + else: + self.part_iter = part_iter + self.comparative = comparative - sort_joined_rows = staticmethod(sort_by_last_column) - def calculate_rows(self, stats: Stats) -> Rows: - opcode_stats = stats.get_opcode_stats(self.prefix) +def calc_execution_count_table(prefix: str) -> RowCalculator: + def calc(stats: Stats) -> Rows: + opcode_stats = stats.get_opcode_stats(prefix) total = 0 counts = [] for name, opcode_stat in opcode_stats.items(): @@ -390,11 +369,25 @@ def calculate_rows(self, stats: Stats) -> Rows: ) return rows + return calc -class PairCountTable(Table): - columns = ("Pair", "Count:", "Self:", "Cumulative:") - def calculate_rows(self, stats: Stats) -> Rows: +def execution_count_section() -> Section: + return Section( + "Execution counts", + "execution counts for all instructions", + [ + Table( + ("Name", "Count:", "Self:", "Cumulative:", "Miss ratio:"), + calc_execution_count_table("opcode"), + join_mode=JoinMode.CHANGE_ONE_COLUMN, + ) + ], + ) + + +def pair_count_section() -> Section: + def calc_pair_count_table(stats: Stats) -> Rows: pair_counts = stats.get_pair_counts("opcode") total = stats.get_total("opcode") @@ -413,68 +406,87 @@ def calculate_rows(self, stats: Stats) -> Rows: return rows + return Section( + "Pair counts", + "Pair counts for top 100 pairs", + [ + Table( + ("Pair", "Count:", "Self:", "Cumulative:"), + calc_pair_count_table, + ) + ], + comparative=False, + ) -def iter_pre_succ_pairs_tables(base_stats: Stats, head_stats: Stats | None = None): - assert head_stats is None - opcode_stats = base_stats.get_opcode_stats("opcode") - pair_counts = base_stats.get_pair_counts("opcode") +def pre_succ_pairs_section() -> Section: + def iter_pre_succ_pairs_tables(base_stats: Stats, head_stats: Stats | None = None): + assert head_stats is None - predecessors: dict[str, collections.Counter] = collections.defaultdict( - collections.Counter - ) - successors: dict[str, collections.Counter] = collections.defaultdict( - collections.Counter - ) - total_predecessors: collections.Counter = collections.Counter() - total_successors: collections.Counter = collections.Counter() - for count, (first, second) in pair_counts: - if count: - predecessors[second][first] = count - successors[first][second] = count - total_predecessors[second] += count - total_successors[first] += count - - for name in opcode_stats.keys(): - total1 = total_predecessors[name] - total2 = total_successors[name] - if total1 == 0 and total2 == 0: - continue - pred_rows: Rows = [] - succ_rows: Rows = [] - if total1: - pred_rows = [ - (pred, Count(count), Ratio(count, total1)) - for (pred, count) in predecessors[name].most_common(5) - ] - if total2: - succ_rows = [ - (succ, Count(count), Ratio(count, total2)) - for (succ, count) in successors[name].most_common(5) - ] + opcode_stats = base_stats.get_opcode_stats("opcode") + pair_counts = base_stats.get_pair_counts("opcode") - yield Section( - name, - f"Successors and predecessors for {name}", - [ - FixedTable(("Predecessors", "Count:", "Percentage:"), pred_rows), - FixedTable(("Successors", "Count:", "Percentage:"), succ_rows), - ], + predecessors: dict[str, collections.Counter] = collections.defaultdict( + collections.Counter ) + successors: dict[str, collections.Counter] = collections.defaultdict( + collections.Counter + ) + total_predecessors: collections.Counter = collections.Counter() + total_successors: collections.Counter = collections.Counter() + for count, (first, second) in pair_counts: + if count: + predecessors[second][first] = count + successors[first][second] = count + total_predecessors[second] += count + total_successors[first] += count + + for name in opcode_stats.keys(): + total1 = total_predecessors[name] + total2 = total_successors[name] + if total1 == 0 and total2 == 0: + continue + pred_rows: Rows = [] + succ_rows: Rows = [] + if total1: + pred_rows = [ + (pred, Count(count), Ratio(count, total1)) + for (pred, count) in predecessors[name].most_common(5) + ] + if total2: + succ_rows = [ + (succ, Count(count), Ratio(count, total2)) + for (succ, count) in successors[name].most_common(5) + ] + yield Section( + name, + f"Successors and predecessors for {name}", + [ + Table( + ("Predecessors", "Count:", "Percentage:"), + lambda *_: pred_rows, + ), + Table( + ("Successors", "Count:", "Percentage:"), + lambda *_: succ_rows, + ), + ], + ) -def iter_specialization_tables(base_stats: Stats, head_stats: Stats | None = None): - class SpecializationTable(SimpleChangeTable): - columns = ("Kind", "Count:", "Ratio:") - - def __init__(self, name: str): - self.name = name + return Section( + "Predecessor/Successor Pairs", + "Top 5 predecessors and successors of each opcode", + iter_pre_succ_pairs_tables, + comparative=False, + ) - sort_joined_rows = staticmethod(dont_sort) - def calculate_rows(self, stats: Stats) -> Rows: +def specialization_section() -> Section: + def calc_specialization_table(name: str) -> RowCalculator: + def calc(stats: Stats) -> Rows: opcode_stats = stats.get_opcode_stats("opcode") - family_stats = opcode_stats[self.name] + family_stats = opcode_stats[name] total = sum(family_stats.get(kind, 0) for kind in TOTAL) if total == 0: return [] @@ -502,17 +514,12 @@ def calculate_rows(self, stats: Stats) -> Rows: ) return rows - class SpecializationSuccessFailureTable(SimpleChangeTable): - columns = ("", "Count:", "Ratio:") - - def __init__(self, name: str): - self.name = name - - sort_joined_rows = staticmethod(dont_sort) + return calc - def calculate_rows(self, stats: Stats) -> Rows: + def calc_specialization_success_failure_table(name: str) -> RowCalculator: + def calc(stats: Stats) -> Rows: opcode_stats = stats.get_opcode_stats("opcode") - family_stats = opcode_stats[self.name] + family_stats = opcode_stats[name] total_attempts = 0 for key in ("specialization.success", "specialization.failure"): total_attempts += family_stats.get(key, 0) @@ -525,13 +532,10 @@ def calculate_rows(self, stats: Stats) -> Rows: rows.append((label, Count(val), Ratio(val, total_attempts))) return rows - class SpecializationFailureKindTable(SimpleChangeTable): - columns = ("Failure kind", "Count:", "Ratio:") + return calc - def __init__(self, name: str): - self.name = name - - def calculate_rows(self, stats: Stats) -> Rows: + def calc_specialization_failure_kind_table(name: str) -> RowCalculator: + def calc(stats: Stats) -> Rows: def kind_to_text(kind: int, defines: Defines, opname: str): if kind <= 8: return pretty(defines[kind][0]) @@ -550,7 +554,7 @@ def kind_to_text(kind: int, defines: Defines, opname: str): defines = stats.defines opcode_stats = stats.get_opcode_stats("opcode") - family_stats = opcode_stats[self.name] + family_stats = opcode_stats[name] total_failures = family_stats.get("specialization.failure", 0) failure_kinds = [0] * 40 for key in family_stats: @@ -566,137 +570,174 @@ def kind_to_text(kind: int, defines: Defines, opname: str): continue rows.append( ( - kind_to_text(index, defines, self.name), + kind_to_text(index, defines, name), Count(value), Ratio(value, total_failures), ) ) return rows - opcode_base_stats = base_stats.get_opcode_stats("opcode") - names = opcode_base_stats.keys() - if head_stats is not None: - opcode_head_stats = head_stats.get_opcode_stats("opcode") - names &= opcode_head_stats.keys() # type: ignore - else: - opcode_head_stats = {} - - for name in sorted(names): - if "specializable" not in opcode_base_stats.get(name, {}): - continue - total = sum( - stats.get(name, {}).get(kind, 0) - for kind in TOTAL - for stats in (opcode_base_stats, opcode_head_stats) - ) - if total == 0: - continue - yield Section( - name, - f"specialization stats for {name} family", - [ - SpecializationTable(name), - SpecializationSuccessFailureTable(name), - SpecializationFailureKindTable(name), - ], - ) - + return calc -def iter_specialization_effectiveness_tables( - base_stats: Stats, head_stats: Stats | None = None -): - class SpecializationEffectivenessTable(SimpleChangeTable): - columns = ("Instructions", "Count:", "Ratio:") - - sort_joined_rows = staticmethod(dont_sort) + def iter_specialization_tables(base_stats: Stats, head_stats: Stats | None = None): + opcode_base_stats = base_stats.get_opcode_stats("opcode") + names = opcode_base_stats.keys() + if head_stats is not None: + opcode_head_stats = head_stats.get_opcode_stats("opcode") + names &= opcode_head_stats.keys() # type: ignore + else: + opcode_head_stats = {} - def calculate_rows(self, stats: Stats) -> Rows: - opcode_stats = stats.get_opcode_stats("opcode") - total = stats.get_total("opcode") - specialized_instructions = stats.specialized_instructions - - basic = 0 - specialized = 0 - not_specialized = 0 - for name, opcode_stat in opcode_stats.items(): - if "execution_count" not in opcode_stat: - continue - count = opcode_stat["execution_count"] - if "specializable" in opcode_stat: - not_specialized += count - elif name in specialized_instructions: - miss = opcode_stat.get("specialization.miss", 0) - not_specialized += miss - specialized += count - miss - else: - basic += count + for name in sorted(names): + if "specializable" not in opcode_base_stats.get(name, {}): + continue + total = sum( + stats.get(name, {}).get(kind, 0) + for kind in TOTAL + for stats in (opcode_base_stats, opcode_head_stats) + ) + if total == 0: + continue + yield Section( + name, + f"specialization stats for {name} family", + [ + Table( + ("Kind", "Count:", "Ratio:"), + calc_specialization_table(name), + JoinMode.CHANGE, + ), + Table( + ("", "Count:", "Ratio:"), + calc_specialization_success_failure_table(name), + JoinMode.CHANGE, + ), + Table( + ("Failure kind", "Count:", "Ratio:"), + calc_specialization_failure_kind_table(name), + JoinMode.CHANGE, + ), + ], + ) - return [ - ("Basic", Count(basic), Ratio(basic, total)), - ( - "Not specialized", - Count(not_specialized), - Ratio(not_specialized, total), - ), - ("Specialized", Count(specialized), Ratio(specialized, total)), - ] + return Section( + "Specialization stats", + "specialization stats by family", + iter_specialization_tables, + ) - class DeferredByInstructionTable(SimpleChangeTable): - columns = ("Name", "Count:", "Ratio:") - def calculate_rows(self, stats: Stats) -> Rows: - opcode_stats = stats.get_opcode_stats("opcode") +def specialization_effectiveness_section() -> Section: + def calc_specialization_effectiveness_table(stats: Stats) -> Rows: + opcode_stats = stats.get_opcode_stats("opcode") + total = stats.get_total("opcode") + specialized_instructions = stats.specialized_instructions - total = 0 - counts = [] - for name, opcode_stat in opcode_stats.items(): - value = opcode_stat.get("specialization.deferred", 0) - counts.append((value, name)) - total += value - counts.sort(reverse=True) - if total: - return [ - (name, Count(count), Ratio(count, total)) - for (count, name) in counts[:10] - ] + basic = 0 + specialized = 0 + not_specialized = 0 + for name, opcode_stat in opcode_stats.items(): + if "execution_count" not in opcode_stat: + continue + count = opcode_stat["execution_count"] + if "specializable" in opcode_stat: + not_specialized += count + elif name in specialized_instructions: + miss = opcode_stat.get("specialization.miss", 0) + not_specialized += miss + specialized += count - miss else: - return [] + basic += count - class MissesByInstructionTable(SimpleChangeTable): - columns = ("Name", "Count:", "Ratio:") + return [ + ("Basic", Count(basic), Ratio(basic, total)), + ( + "Not specialized", + Count(not_specialized), + Ratio(not_specialized, total), + ), + ("Specialized", Count(specialized), Ratio(specialized, total)), + ] - def calculate_rows(self, stats: Stats) -> Rows: - opcode_stats = stats.get_opcode_stats("opcode") + def calc_deferred_by_table(stats: Stats) -> Rows: + opcode_stats = stats.get_opcode_stats("opcode") - total = 0 - counts = [] - for name, opcode_stat in opcode_stats.items(): - # Avoid double counting misses - if "specializable" in opcode_stat: - continue - value = opcode_stat.get("specialization.misses", 0) - counts.append((value, name)) - total += value - counts.sort(reverse=True) - if total: - return [ - (name, Count(count), Ratio(count, total)) - for (count, name) in counts[:10] - ] - else: - return [] + total = 0 + counts = [] + for name, opcode_stat in opcode_stats.items(): + value = opcode_stat.get("specialization.deferred", 0) + counts.append((value, name)) + total += value + counts.sort(reverse=True) + if total: + return [ + (name, Count(count), Ratio(count, total)) + for (count, name) in counts[:10] + ] + else: + return [] - yield SpecializationEffectivenessTable() - yield Section("Deferred by instruction", "", [DeferredByInstructionTable()]) - yield Section("Misses by instruction", "", [MissesByInstructionTable()]) + def calc_misses_by_table(stats: Stats) -> Rows: + opcode_stats = stats.get_opcode_stats("opcode") + + total = 0 + counts = [] + for name, opcode_stat in opcode_stats.items(): + # Avoid double counting misses + if "specializable" in opcode_stat: + continue + value = opcode_stat.get("specialization.misses", 0) + counts.append((value, name)) + total += value + counts.sort(reverse=True) + if total: + return [ + (name, Count(count), Ratio(count, total)) + for (count, name) in counts[:10] + ] + else: + return [] + def iter_specialization_effectiveness_tables( + base_stats: Stats, head_stats: Stats | None = None + ): + yield Table( + ("Instructions", "Count:", "Ratio:"), + calc_specialization_effectiveness_table, + JoinMode.CHANGE, + ) + yield Section( + "Deferred by instruction", + "", + [ + Table( + ("Name", "Count:", "Ratio:"), + calc_deferred_by_table, + JoinMode.CHANGE, + ) + ], + ) + yield Section( + "Misses by instruction", + "", + [ + Table( + ("Name", "Count:", "Ratio:"), + calc_misses_by_table, + JoinMode.CHANGE, + ) + ], + ) -class CallStatsTable(SimpleChangeTable): - columns = ("", "Count:", "Ratio:") + return Section( + "Specialization effectiveness", + "", + iter_specialization_effectiveness_tables, + ) - sort_joined_rows = staticmethod(dont_sort) - def calculate_rows(self, stats: Stats) -> Rows: +def call_stats_section() -> Section: + def calc_call_stats_table(stats: Stats) -> Rows: defines = stats.pystats_defines total = 0 @@ -720,11 +761,21 @@ def calculate_rows(self, stats: Stats) -> Rows: return rows + return Section( + "Call stats", + "Inlined calls and frame stats", + [ + Table( + ("", "Count:", "Ratio:"), + calc_call_stats_table, + JoinMode.CHANGE, + ) + ], + ) -class ObjectStatsTable(SimpleChangeTable): - columns = ("", "Count:", "Ratio:") - def calculate_rows(self, stats: Stats) -> Rows: +def object_stats_section() -> Section: + def calc_object_stats_table(stats: Stats) -> Rows: total_materializations = stats.get("Object new values", 0) total_allocations = stats.get("Object allocations", 0) + stats.get( "Object allocations from freelist", 0 @@ -753,11 +804,21 @@ def calculate_rows(self, stats: Stats) -> Rows: rows.append((label, Count(value), ratio)) return rows + return Section( + "Object stats", + "allocations, frees and dict materializatons", + [ + Table( + ("", "Count:", "Ratio:"), + calc_object_stats_table, + JoinMode.CHANGE, + ) + ], + ) -class GCStatsTable(Table): - columns = ("Generation:", "Collections:", "Objects collected:", "Object visits:") - def calculate_rows(self, stats: Stats) -> Rows: +def gc_stats_section() -> Section: + def calc_gc_stats(stats: Stats) -> Rows: gc_stats: list[dict[str, int]] = [] for key, value in stats.items(): if not key.startswith("GC"): @@ -778,239 +839,260 @@ def calculate_rows(self, stats: Stats) -> Rows: for (i, gen) in enumerate(gc_stats) ] + return Section( + "GC stats", + "GC collections and effectiveness", + [ + Table( + ("Generation:", "Collections:", "Objects collected:", "Object visits:"), + calc_gc_stats, + ) + ], + ) -def iter_optimization_tables(base_stats: Stats, head_stats: Stats | None = None): - class OptimizationStatsTable(SimpleChangeTable): - columns = ("", "Count:", "Ratio:") - - sort_joined_rows = staticmethod(dont_sort) - - def calculate_rows(self, stats: Stats) -> Rows: - if "Optimization attempts" not in stats: - return [] - - attempts = stats["Optimization attempts"] - created = stats["Optimization traces created"] - executed = stats["Optimization traces executed"] - uops = stats["Optimization uops executed"] - trace_stack_overflow = stats["Optimization trace stack overflow"] - trace_stack_underflow = stats["Optimization trace stack underflow"] - trace_too_long = stats["Optimization trace too long"] - trace_too_short = stats["Optimization trace too short"] - inner_loop = stats["Optimization inner loop"] - recursive_call = stats["Optimization recursive call"] - - return [ - ("Optimization attempts", Count(attempts), ""), - ("Traces created", Count(created), Ratio(created, attempts)), - ("Traces executed", Count(executed), ""), - ("Uops executed", Count(uops), CountPer(uops, executed)), - ( - "Trace stack overflow", - Count(trace_stack_overflow), - Ratio(trace_stack_overflow, created), - ), - ( - "Trace stack underflow", - Count(trace_stack_underflow), - Ratio(trace_stack_underflow, created), - ), - ( - "Trace too long", - Count(trace_too_long), - Ratio(trace_too_long, created), - ), - ( - "Trace too short", - Count(trace_too_short), - Ratio(trace_too_short, created), - ), - ("Inner loop found", Count(inner_loop), Ratio(inner_loop, created)), - ( - "Recursive call", - Count(recursive_call), - Ratio(recursive_call, created), - ), - ] - class HistogramTable(SimpleChangeTable): - columns = ("Range", "Count:", "Ratio:") +def optimization_section() -> Section: + def calc_optimization_table(stats: Stats) -> Rows: + if "Optimization attempts" not in stats: + return [] - def __init__(self, key: str, den: str): - self.key = key - self.den = den + attempts = stats["Optimization attempts"] + created = stats["Optimization traces created"] + executed = stats["Optimization traces executed"] + uops = stats["Optimization uops executed"] + trace_stack_overflow = stats["Optimization trace stack overflow"] + trace_stack_underflow = stats["Optimization trace stack underflow"] + trace_too_long = stats["Optimization trace too long"] + trace_too_short = stats["Optimization trace too short"] + inner_loop = stats["Optimization inner loop"] + recursive_call = stats["Optimization recursive call"] - sort_joined_rows = staticmethod(dont_sort) + return [ + ("Optimization attempts", Count(attempts), ""), + ("Traces created", Count(created), Ratio(created, attempts)), + ("Traces executed", Count(executed), ""), + ("Uops executed", Count(uops), Ratio(uops, executed, percentage=False)), + ( + "Trace stack overflow", + Count(trace_stack_overflow), + Ratio(trace_stack_overflow, created), + ), + ( + "Trace stack underflow", + Count(trace_stack_underflow), + Ratio(trace_stack_underflow, created), + ), + ( + "Trace too long", + Count(trace_too_long), + Ratio(trace_too_long, created), + ), + ( + "Trace too short", + Count(trace_too_short), + Ratio(trace_too_short, created), + ), + ("Inner loop found", Count(inner_loop), Ratio(inner_loop, created)), + ( + "Recursive call", + Count(recursive_call), + Ratio(recursive_call, created), + ), + ] - def calculate_rows(self, stats: Stats) -> Rows: + def calc_histogram_table(key: str, den: str) -> RowCalculator: + def calc(stats: Stats) -> Rows: rows: Rows = [] last_non_zero = 0 for k, v in stats.items(): - if k.startswith(self.key): + if k.startswith(key): match = re.match(r".+\[([0-9]+)\]", k) if match is not None: entry = int(match.groups()[0]) if v != 0: last_non_zero = len(rows) rows.append( - (f"<= {entry:,d}", Count(v), Ratio(int(v), stats[self.den])) + ( + f"<= {entry:,d}", + Count(v), + Ratio(int(v), stats[den]), + ) ) # Don't include any zero entries at the end rows = rows[: last_non_zero + 1] return rows - class UnsupportedOpcodesTable(SimpleChangeTable): - columns = ("Opcode", "Count:") - - def calculate_rows(self, stats: Stats) -> Rows: - unsupported_opcodes = stats.get_opcode_stats("unsupported_opcode") - data = [] - for opcode, entry in unsupported_opcodes.items(): - data.append((Count(entry["count"]), opcode)) - data.sort(reverse=True) - return [(x[1], x[0]) for x in data] - - if "Optimization attempts" not in base_stats or ( - head_stats is not None and "Optimization attempts" not in head_stats - ): - return + return calc - yield OptimizationStatsTable() + def calc_unsupported_opcodes_table(stats: Stats) -> Rows: + unsupported_opcodes = stats.get_opcode_stats("unsupported_opcode") + data = [] + for opcode, entry in unsupported_opcodes.items(): + data.append((Count(entry["count"]), opcode)) + data.sort(reverse=True) + return [(x[1], x[0]) for x in data] - for name, den in [ - ("Trace length", "Optimization traces created"), - ("Optimized trace length", "Optimization traces created"), - ("Trace run length", "Optimization traces executed"), - ]: - yield Section(f"{name} histogram", "", [HistogramTable(name, den)]) + def iter_optimization_tables(base_stats: Stats, head_stats: Stats | None = None): + if "Optimization attempts" not in base_stats or ( + head_stats is not None and "Optimization attempts" not in head_stats + ): + return - yield Section("Uop stats", "", [ExecutionCountTable("uops")]) - yield Section("Unsupported opcodes", "", [UnsupportedOpcodesTable()]) + yield Table(("", "Count:", "Ratio:"), calc_optimization_table, JoinMode.CHANGE) + for name, den in [ + ("Trace length", "Optimization traces created"), + ("Optimized trace length", "Optimization traces created"), + ("Trace run length", "Optimization traces executed"), + ]: + yield Section( + f"{name} histogram", + "", + [ + Table( + ("Range", "Count:", "Ratio:"), + calc_histogram_table(name, den), + JoinMode.CHANGE, + ) + ], + ) + yield Section( + "Uop stats", + "", + [ + Table( + ("Name", "Count:", "Self:", "Cumulative:", "Miss ratio:"), + calc_execution_count_table("uops"), + JoinMode.CHANGE_ONE_COLUMN, + ) + ], + ) + yield Section( + "Unsupported opcodes", + "", + [ + Table( + ("Opcode", "Count:"), + calc_unsupported_opcodes_table, + JoinMode.CHANGE, + ) + ], + ) + return Section( + "Optimization (Tier 2) stats", + "statistics about the Tier 2 optimizer", + iter_optimization_tables, + ) -class MetaStatsTable(Table): - columns = ("", "Count:") - def calculate_rows(self, stats: Stats) -> Rows: +def meta_stats_section() -> Section: + def calc_rows(stats: Stats) -> Rows: return [("Number of data files", Count(stats.get("__nfiles__", 0)))] + return Section( + "Meta stats", + "Meta statistics", + [Table(("", "Count:"), calc_rows, JoinMode.CHANGE)], + ) -class Section: - def __init__( - self, - title: str, - summary: str = "", - part_iter=None, - comparative: bool = True, - ): - self.title = title - if not summary: - self.summary = title.lower() - else: - self.summary = summary - if part_iter is None: - part_iter = [] - if isinstance(part_iter, list): - - def iter_parts(base_stats: Stats, head_stats: Stats | None): - yield from part_iter - self.part_iter = iter_parts +LAYOUT = [ + execution_count_section(), + pair_count_section(), + pre_succ_pairs_section(), + specialization_section(), + specialization_effectiveness_section(), + call_stats_section(), + gc_stats_section(), + optimization_section(), + meta_stats_section(), +] + + +def output_markdown( + out: TextIO, + obj: Section | Table | list, + base_stats: Stats, + head_stats: Stats | None = None, + level: int = 2, +) -> None: + def to_markdown(x): + if hasattr(x, "markdown"): + return x.markdown() + elif isinstance(x, str): + return x + elif x is None: + return "" else: - self.part_iter = part_iter - self.comparative = comparative - - def output_markdown( - self, - out: TextIO, - base_stats: Stats, - head_stats: Stats | None = None, - level: int = 1, - ) -> None: - if self.title: - print("#" * level, self.title, file=out) - print(file=out) - print("
", file=out) - print("", self.summary, "", file=out) + raise TypeError(f"Can't convert {x} to markdown") + + match obj: + case Section(): + if obj.title: + print("#" * level, obj.title, file=out) + print(file=out) + print("
", file=out) + print("", obj.summary, "", file=out) + print(file=out) + if head_stats is not None and obj.comparative is False: + print("Not included in comparative output.\n") + else: + for part in obj.part_iter(base_stats, head_stats): + output_markdown(out, part, base_stats, head_stats, level=level + 1) print(file=out) - if head_stats is not None and self.comparative is False: - print("Not included in comparative output.\n") - else: - for part in self.part_iter(base_stats, head_stats): - part.output_markdown(out, base_stats, head_stats, level=level + 1) - print(file=out) - if self.title: - print("
", file=out) + if obj.title: + print("
", file=out) + print(file=out) + + case Table(): + header, rows = obj.get_table(base_stats, head_stats) + if len(rows) == 0: + return + + width = len(header) + header_line = "|" + under_line = "|" + for item in header: + under = "---" + if item.endswith(":"): + item = item[:-1] + under += ":" + header_line += item + " | " + under_line += under + "|" + print(header_line, file=out) + print(under_line, file=out) + for row in rows: + if len(row) != width: + raise ValueError( + "Wrong number of elements in row '" + str(row) + "'" + ) + print("|", " | ".join(to_markdown(i) for i in row), "|", file=out) print(file=out) + case list(): + for part in obj: + output_markdown(out, part, base_stats, head_stats, level=level) -LAYOUT = Section( - "", - "", - [ - Section( - "Execution counts", - "execution counts for all instructions", - [ExecutionCountTable("opcode")], - ), - Section( - "Pair counts", - "Pair counts for top 100 pairs", - [PairCountTable()], - comparative=False, - ), - Section( - "Predecessor/Successor Pairs", - "Top 5 predecessors and successors of each opcode", - iter_pre_succ_pairs_tables, - comparative=False, - ), - Section( - "Specialization stats", - "specialization stats by family", - iter_specialization_tables, - ), - Section( - "Specialization effectiveness", - "", - iter_specialization_effectiveness_tables, - ), - Section("Call stats", "Inlined calls and frame stats", [CallStatsTable()]), - Section( - "Object stats", - "allocations, frees and dict materializatons", - [ObjectStatsTable()], - ), - Section("GC stats", "GC collections and effectiveness", [GCStatsTable()]), - Section( - "Optimization (Tier 2) stats", - "statistics about the Tier 2 optimizer", - iter_optimization_tables, - ), - Section("Meta stats", "Meta statistics", [MetaStatsTable()]), - ], -) - - -def output_markdown(out: TextIO, base_stats: Stats, head_stats: Stats | None = None): - LAYOUT.output_markdown(out, base_stats, head_stats) - print("---", file=out) - print("Stats gathered on:", date.today(), file=out) + print("---", file=out) + print("Stats gathered on:", date.today(), file=out) def output_stats(inputs: list[Path], json_output=TextIO | None): - if len(inputs) == 1: - stats = Stats(Path(inputs[0])) - if json_output is not None: - stats.save(json_output) - output_markdown(sys.stdout, stats) - elif len(inputs) == 2: - if json_output is not None: - raise ValueError("Can not output to JSON when there are multiple inputs") - - base_stats = Stats(Path(inputs[0])) - head_stats = Stats(Path(inputs[1])) - output_markdown(sys.stdout, base_stats, head_stats) + match len(inputs): + case 1: + stats = Stats(Path(inputs[0])) + if json_output is not None: + stats.save(json_output) # type: ignore + output_markdown(sys.stdout, LAYOUT, stats) + case 2: + if json_output is not None: + raise ValueError( + "Can not output to JSON when there are multiple inputs" + ) + + base_stats = Stats(Path(inputs[0])) + head_stats = Stats(Path(inputs[1])) + output_markdown(sys.stdout, LAYOUT, base_stats, head_stats) def main(): From 582b1b63b08a0ea2d94cafd923d0351240ad3fc1 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Fri, 6 Oct 2023 14:31:06 -0400 Subject: [PATCH 6/8] Separate data processing from display --- Tools/scripts/summarize_stats.py | 988 ++++++++++++++++--------------- 1 file changed, 523 insertions(+), 465 deletions(-) diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index ed38ff201e7924..7d60995bee9a26 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -10,25 +10,29 @@ import argparse import collections +from collections.abc import KeysView from datetime import date import enum +import functools import itertools import json +from operator import itemgetter import os from pathlib import Path import re import sys -from typing import Callable, TextIO, TypeAlias +from typing import Any, Callable, TextIO, TypeAlias -OpcodeStats: TypeAlias = dict[str, dict[str, int]] -PairCounts: TypeAlias = list[tuple[int, tuple[str, str]]] -Defines: TypeAlias = dict[int, list[str]] +RawData: TypeAlias = dict[str, Any] Rows: TypeAlias = list[tuple] Columns: TypeAlias = tuple[str, ...] RowCalculator: TypeAlias = Callable[["Stats"], Rows] +# TODO: Check for parity + + if os.name == "nt": DEFAULT_DIR = "c:\\temp\\py_stats\\" else: @@ -45,135 +49,363 @@ def pretty(name: str) -> str: return name.replace("_", " ").lower() -class Stats: - _data: dict - - def __init__(self, input: Path): - super().__init__() - - if input.is_file(): - with open(input, "r") as fd: - self._data = json.load(fd) - - self._data["_stats_defines"] = { - int(k): v for k, v in self["_stats_defines"].items() - } - self._data["_defines"] = {int(k): v for k, v in self["_defines"].items()} - - elif input.is_dir(): - stats = collections.Counter[str]() - - for filename in input.iterdir(): - with open(filename) as fd: - for line in fd: - try: - key, value = line.split(":") - except ValueError: - print( - f"Unparsable line: '{line.strip()}' in {filename}", - file=sys.stderr, - ) - continue - stats[key.strip()] += int(value) - stats["__nfiles__"] += 1 - - self._data = dict(stats) - self._load_metadata_from_source() +def _load_metadata_from_source(): + def get_defines(filepath: Path, prefix: str = "SPEC_FAIL"): + with open(SOURCE_DIR / filepath) as spec_src: + defines = collections.defaultdict(list) + start = "#define " + prefix + "_" + for line in spec_src: + line = line.strip() + if not line.startswith(start): + continue + line = line[len(start) :] + name, val = line.split() + defines[int(val.strip())].append(name.strip()) + return defines + + import opcode + + return { + "_specialized_instructions": [ + op for op in opcode._specialized_opmap.keys() if "__" not in op # type: ignore + ], + "_stats_defines": get_defines( + Path("Include") / "cpython" / "pystats.h", "EVAL_CALL" + ), + "_defines": get_defines(Path("Python") / "specialize.c"), + } - else: - raise ValueError(f"{input:r} is not a file or directory path") - self._opcode_stats: dict[str, OpcodeStats] = {} +def load_raw_data(input: Path) -> RawData: + if input.is_file(): + with open(input, "r") as fd: + data = json.load(fd) - def __getitem__(self, key): - return self._data[key] + data["_stats_defines"] = {int(k): v for k, v in data["_stats_defines"].items()} + data["_defines"] = {int(k): v for k, v in data["_defines"].items()} - def __contains__(self, key): - return key in self._data + return data - def get(self, key, default=None): - return self._data.get(key, default) + elif input.is_dir(): + stats = collections.Counter[str]() + + for filename in input.iterdir(): + with open(filename) as fd: + for line in fd: + try: + key, value = line.split(":") + except ValueError: + print( + f"Unparsable line: '{line.strip()}' in {filename}", + file=sys.stderr, + ) + continue + stats[key.strip()] += int(value) + stats["__nfiles__"] += 1 + + data = dict(stats) + data.update(_load_metadata_from_source()) + return data + + else: + raise ValueError(f"{input:r} is not a file or directory path") - def items(self): - return self._data.items() - def keys(self): +def save_raw_data(data: RawData, json_output: TextIO): + json.dump(data, json_output) + + +class OpcodeStats: + """ + Manages the data related to specific set of opcodes, e.g. tier1 (with prefix + "opcode") or tier2 (with prefix "uops"). + """ + + def __init__(self, data: dict[str, Any], defines, specialized_instructions): + self._data = data + self._defines = defines + self._specialized_instructions = specialized_instructions + + def get_opcode_names(self) -> KeysView[str]: return self._data.keys() - def values(self): - return self._data.values() + def get_pair_counts(self) -> dict[tuple[str, str], int]: + pair_counts = {} + for name_i, opcode_stat in self._data.items(): + for key, value in opcode_stat.items(): + if value and key.startswith("pair_count"): + name_j, _, _ = key[len("pair_count") + 1 :].partition("]") + pair_counts[(name_i, name_j)] = value + return pair_counts - def save(self, json_output: TextIO): - json.dump(self._data, json_output) + def get_total_execution_count(self) -> int: + return sum(x.get("execution_count", 0) for x in self._data.values()) - def _load_metadata_from_source(self): - def get_defines(filepath: Path, prefix: str = "SPEC_FAIL") -> Defines: - with open(SOURCE_DIR / filepath) as spec_src: - defines = collections.defaultdict(list) - start = "#define " + prefix + "_" - for line in spec_src: - line = line.strip() - if not line.startswith(start): - continue - line = line[len(start) :] - name, val = line.split() - defines[int(val.strip())].append(name.strip()) - return defines + def get_execution_counts(self) -> dict[str, tuple[int, int]]: + counts = {} + for name, opcode_stat in self._data.items(): + if "execution_count" in opcode_stat: + count = opcode_stat["execution_count"] + miss = 0 + if "specializable" not in opcode_stat: + miss = opcode_stat.get("specialization.miss", 0) + counts[name] = (count, miss) + return counts - import opcode + @functools.cache + def _get_pred_succ( + self, + ) -> tuple[dict[str, collections.Counter], dict[str, collections.Counter]]: + pair_counts = self.get_pair_counts() - self._data["_specialized_instructions"] = [ - op - for op in opcode._specialized_opmap.keys() # type: ignore - if "__" not in op - ] - self._data["_stats_defines"] = get_defines( - Path("Include") / "cpython" / "pystats.h", "EVAL_CALL" + predecessors: dict[str, collections.Counter] = collections.defaultdict( + collections.Counter + ) + successors: dict[str, collections.Counter] = collections.defaultdict( + collections.Counter ) - self._data["_defines"] = get_defines(Path("Python") / "specialize.c") + for (first, second), count in pair_counts.items(): + if count: + predecessors[second][first] = count + successors[first][second] = count - @property - def defines(self) -> Defines: - return self._data["_defines"] + return predecessors, successors - @property - def pystats_defines(self) -> Defines: - return self._data["_stats_defines"] + def get_predecessors(self, opcode: str) -> collections.Counter[str]: + return self._get_pred_succ()[0][opcode] - @property - def specialized_instructions(self) -> list[str]: - return self._data["_specialized_instructions"] + def get_successors(self, opcode: str) -> collections.Counter[str]: + return self._get_pred_succ()[1][opcode] - def get_opcode_stats(self, prefix: str) -> OpcodeStats: - if prefix in self._opcode_stats: - return self._opcode_stats[prefix] + def _get_stats_for_opcode(self, opcode: str) -> dict[str, int]: + return self._data[opcode] + + def get_specialization_total(self, opcode: str) -> int: + family_stats = self._get_stats_for_opcode(opcode) + return sum(family_stats.get(kind, 0) for kind in TOTAL) - opcode_stats: OpcodeStats = collections.defaultdict(dict) - for key, value in self.items(): + def get_specialization_counts(self, opcode: str) -> dict[str, int]: + family_stats = self._get_stats_for_opcode(opcode) + + result = {} + for key, value in sorted(family_stats.items()): + if key.startswith("specialization."): + label = key[len("specialization.") :] + if label in ("success", "failure") or label.startswith("failure_kinds"): + continue + elif key in ( + "execution_count", + "specializable", + ) or key.startswith("pair"): + continue + else: + label = key + result[label] = value + + return result + + def get_specialization_success_failure(self, opcode: str) -> dict[str, int]: + family_stats = self._get_stats_for_opcode(opcode) + result = {} + for key in ("specialization.success", "specialization.failure"): + label = key[len("specialization.") :] + val = family_stats.get(key, 0) + result[label] = val + return result + + def get_specialization_failure_total(self, opcode: str) -> int: + return self._get_stats_for_opcode(opcode).get("specialization.failure", 0) + + def get_specialization_failure_kinds(self, opcode: str) -> dict[str, int]: + def kind_to_text(kind: int, opcode: str): + if kind <= 8: + return pretty(self._defines[kind][0]) + if opcode == "LOAD_SUPER_ATTR": + opcode = "SUPER" + elif opcode.endswith("ATTR"): + opcode = "ATTR" + elif opcode in ("FOR_ITER", "SEND"): + opcode = "ITER" + elif opcode.endswith("SUBSCR"): + opcode = "SUBSCR" + for name in self._defines[kind]: + if name.startswith(opcode): + return pretty(name[len(opcode) + 1 :]) + return "kind " + str(kind) + + family_stats = self._get_stats_for_opcode(opcode) + failure_kinds = [0] * 40 + for key in family_stats: + if not key.startswith("specialization.failure_kind"): + continue + index = int(key[:-1].split("[")[1]) + failure_kinds[index] = family_stats[key] + return { + kind_to_text(index, opcode): value + for (index, value) in enumerate(failure_kinds) + if value + } + + def is_specializable(self, opcode: str) -> bool: + return "specializable" in self._get_stats_for_opcode(opcode) + + def get_specialized_total_counts(self) -> tuple[int, int, int]: + basic = 0 + specialized = 0 + not_specialized = 0 + for opcode, opcode_stat in self._data.items(): + if "execution_count" not in opcode_stat: + continue + count = opcode_stat["execution_count"] + if "specializable" in opcode_stat: + not_specialized += count + elif opcode in self._specialized_instructions: + miss = opcode_stat.get("specialization.miss", 0) + not_specialized += miss + specialized += count - miss + else: + basic += count + return basic, specialized, not_specialized + + def get_deferred_counts(self) -> dict[str, int]: + return { + opcode: opcode_stat.get("specialization.deferred", 0) + for opcode, opcode_stat in self._data.items() + } + + def get_misses_counts(self) -> dict[str, int]: + return { + opcode: opcode_stat.get("specialization.miss", 0) + for opcode, opcode_stat in self._data.items() + if not self.is_specializable(opcode) + } + + def get_opcode_counts(self) -> dict[str, int]: + counts = {} + for opcode, entry in self._data.items(): + count = entry.get("count", 0) + if count: + counts[opcode] = count + return counts + + +class Stats: + def __init__(self, data: RawData): + self._data = data + + def get(self, key: str) -> int: + return self._data.get(key, 0) + + @functools.cache + def get_opcode_stats(self, prefix: str) -> OpcodeStats: + opcode_stats = collections.defaultdict[str, dict](dict) + for key, value in self._data.items(): if not key.startswith(prefix): continue name, _, rest = key[len(prefix) + 1 :].partition("]") opcode_stats[name][rest.strip(".")] = value + return OpcodeStats( + opcode_stats, + self._data["_defines"], + self._data["_specialized_instructions"], + ) + + def get_call_stats(self) -> dict[str, int]: + defines = self._data["_stats_defines"] + result = {} + for key, value in sorted(self._data.items()): + if "Calls to" in key: + result[key] = value + elif key.startswith("Calls "): + name, index = key[:-1].split("[") + label = f"{name} ({pretty(defines[int(index)][0])})" + result[label] = value - self._opcode_stats[prefix] = opcode_stats - return opcode_stats + for key, value in sorted(self._data.items()): + if key.startswith("Frame"): + result[key] = value - def get_pair_counts(self, prefix: str) -> PairCounts: - opcode_stats = self.get_opcode_stats(prefix) - pair_counts: PairCounts = [] - for name_i, opcode_stat in opcode_stats.items(): - for key, value in opcode_stat.items(): - if key.startswith("pair_count"): - name_j, _, _ = key[len("pair_count") + 1 :].partition("]") - if value: - pair_counts.append((value, (name_i, name_j))) - pair_counts.sort(reverse=True) - return pair_counts + return result - def get_total(self, prefix: str) -> int: - return sum( - x.get("execution_count", 0) for x in self.get_opcode_stats(prefix).values() + def get_object_stats(self) -> dict[str, tuple[int, int]]: + total_materializations = self._data.get("Object new values", 0) + total_allocations = self._data.get("Object allocations", 0) + self._data.get( + "Object allocations from freelist", 0 ) + total_increfs = self._data.get( + "Object interpreter increfs", 0 + ) + self._data.get("Object increfs", 0) + total_decrefs = self._data.get( + "Object interpreter decrefs", 0 + ) + self._data.get("Object decrefs", 0) + + result = {} + for key, value in self._data.items(): + if key.startswith("Object"): + if "materialize" in key: + den = total_materializations + elif "allocations" in key: + den = total_allocations + elif "increfs" in key: + den = total_increfs + elif "decrefs" in key: + den = total_decrefs + else: + den = None + label = key[6:].strip() + label = label[0].upper() + label[1:] + result[label] = (value, den) + return result + + def get_gc_stats(self) -> list[dict[str, int]]: + gc_stats: list[dict[str, int]] = [] + for key, value in self._data.items(): + if not key.startswith("GC"): + continue + n, _, rest = key[3:].partition("]") + name = rest.strip() + gen_n = int(n) + while len(gc_stats) <= gen_n: + gc_stats.append({}) + gc_stats[gen_n][name] = value + return gc_stats + + def get_optimization_stats(self) -> dict[str, tuple[int, int | None]]: + if "Optimization attempts" not in self._data: + return {} + + attempts = self._data["Optimization attempts"] + created = self._data["Optimization traces created"] + executed = self._data["Optimization traces executed"] + uops = self._data["Optimization uops executed"] + trace_stack_overflow = self._data["Optimization trace stack overflow"] + trace_stack_underflow = self._data["Optimization trace stack underflow"] + trace_too_long = self._data["Optimization trace too long"] + trace_too_short = self._data["Optimization trace too short"] + inner_loop = self._data["Optimization inner loop"] + recursive_call = self._data["Optimization recursive call"] + + return { + "Optimization attempts": (attempts, None), + "Traces created": (created, attempts), + "Traces executed": (executed, None), + "Uops executed": (uops, executed), + "Trace stack overflow": (trace_stack_overflow, created), + "Trace stack underflow": (trace_stack_underflow, created), + "Trace too long": (trace_too_long, created), + "Trace too short": (trace_too_short, created), + "Inner loop found": (inner_loop, created), + "Recursive call": (recursive_call, created), + } + + def get_histogram(self, prefix: str) -> list[tuple[int, int]]: + rows: Rows = [] + for k, v in self._data.items(): + match = re.match(f"{prefix}\\[([0-9]+)\\]", k) + if match is not None: + entry = int(match.groups()[0]) + rows.append((entry, v)) + return sorted(rows) class Count(int): @@ -182,7 +414,7 @@ def markdown(self) -> str: class Ratio: - def __init__(self, num: int, den: int, percentage: bool = True): + def __init__(self, num: int, den: int | None, percentage: bool = True): self.num = num self.den = den self.percentage = percentage @@ -192,11 +424,13 @@ def __init__(self, num: int, den: int, percentage: bool = True): def __float__(self): if self.den == 0: return 0.0 + elif self.den is None: + return self.num else: return self.num / self.den def markdown(self) -> str: - if self.den == 0: + if self.den == 0 or self.den is None: return "" elif self.percentage: return f"{self.num / self.den:,.01%}" @@ -339,20 +573,13 @@ def iter_parts(base_stats: Stats, head_stats: Stats | None): def calc_execution_count_table(prefix: str) -> RowCalculator: def calc(stats: Stats) -> Rows: opcode_stats = stats.get_opcode_stats(prefix) - total = 0 - counts = [] - for name, opcode_stat in opcode_stats.items(): - if "execution_count" in opcode_stat: - count = opcode_stat["execution_count"] - total += count - miss = 0 - if "specializable" not in opcode_stat: - miss = opcode_stat.get("specialization.miss", 0) - counts.append((count, name, miss)) - counts.sort(reverse=True) + counts = opcode_stats.get_execution_counts() + total = opcode_stats.get_total_execution_count() cumulative = 0 rows: Rows = [] - for count, name, miss in counts: + for opcode, (count, miss) in sorted( + counts.items(), key=itemgetter(1), reverse=True + ): cumulative += count if miss: miss_val = Ratio(miss, count) @@ -360,7 +587,7 @@ def calc(stats: Stats) -> Rows: miss_val = None rows.append( ( - name, + opcode, Count(count), Ratio(count, total), Ratio(cumulative, total), @@ -388,22 +615,24 @@ def execution_count_section() -> Section: def pair_count_section() -> Section: def calc_pair_count_table(stats: Stats) -> Rows: - pair_counts = stats.get_pair_counts("opcode") - total = stats.get_total("opcode") + opcode_stats = stats.get_opcode_stats("opcode") + pair_counts = opcode_stats.get_pair_counts() + total = opcode_stats.get_total_execution_count() cumulative = 0 rows: Rows = [] - for count, (name_i, name_j) in itertools.islice(pair_counts, 100): + for (opcode_i, opcode_j), count in itertools.islice( + sorted(pair_counts.items(), key=itemgetter(1), reverse=True), 100 + ): cumulative += count rows.append( ( - f"{name_i} {name_j}", + f"{opcode_i} {opcode_j}", Count(count), Ratio(count, total), Ratio(cumulative, total), ) ) - return rows return Section( @@ -424,52 +653,34 @@ def iter_pre_succ_pairs_tables(base_stats: Stats, head_stats: Stats | None = Non assert head_stats is None opcode_stats = base_stats.get_opcode_stats("opcode") - pair_counts = base_stats.get_pair_counts("opcode") - - predecessors: dict[str, collections.Counter] = collections.defaultdict( - collections.Counter - ) - successors: dict[str, collections.Counter] = collections.defaultdict( - collections.Counter - ) - total_predecessors: collections.Counter = collections.Counter() - total_successors: collections.Counter = collections.Counter() - for count, (first, second) in pair_counts: - if count: - predecessors[second][first] = count - successors[first][second] = count - total_predecessors[second] += count - total_successors[first] += count - for name in opcode_stats.keys(): - total1 = total_predecessors[name] - total2 = total_successors[name] - if total1 == 0 and total2 == 0: + for opcode in opcode_stats.get_opcode_names(): + predecessors = opcode_stats.get_predecessors(opcode) + successors = opcode_stats.get_successors(opcode) + predecessors_total = predecessors.total() + successors_total = successors.total() + if predecessors_total == 0 and successors_total == 0: continue - pred_rows: Rows = [] - succ_rows: Rows = [] - if total1: - pred_rows = [ - (pred, Count(count), Ratio(count, total1)) - for (pred, count) in predecessors[name].most_common(5) - ] - if total2: - succ_rows = [ - (succ, Count(count), Ratio(count, total2)) - for (succ, count) in successors[name].most_common(5) - ] + pred_rows = [ + (pred, Count(count), Ratio(count, predecessors_total)) + for (pred, count) in predecessors.most_common(5) + ] + succ_rows = [ + (succ, Count(count), Ratio(count, successors_total)) + for (succ, count) in successors.most_common(5) + ] yield Section( - name, - f"Successors and predecessors for {name}", + opcode, + f"Successors and predecessors for {opcode}", [ Table( ("Predecessors", "Count:", "Percentage:"), - lambda *_: pred_rows, + lambda *_: pred_rows, # type: ignore ), Table( ("Successors", "Count:", "Percentage:"), - lambda *_: succ_rows, + lambda *_: succ_rows, # type: ignore ), ], ) @@ -483,138 +694,91 @@ def iter_pre_succ_pairs_tables(base_stats: Stats, head_stats: Stats | None = Non def specialization_section() -> Section: - def calc_specialization_table(name: str) -> RowCalculator: + def calc_specialization_table(opcode: str) -> RowCalculator: def calc(stats: Stats) -> Rows: opcode_stats = stats.get_opcode_stats("opcode") - family_stats = opcode_stats[name] - total = sum(family_stats.get(kind, 0) for kind in TOTAL) - if total == 0: - return [] - rows: Rows = [] - for key in sorted(family_stats): - if key.startswith("specialization.failure_kinds"): - continue - elif key in ("specialization.hit", "specialization.miss"): - label = key[len("specialization.") :] - elif key in ( - "execution_count", - "specialization.success", - "specialization.failure", - "specializable", - ) or key.startswith("pair"): - continue - else: - label = key - rows.append( - ( - f"{label:>12}", - Count(family_stats[key]), - Ratio(family_stats[key], total), - ) + total = opcode_stats.get_specialization_total(opcode) + specialization_counts = opcode_stats.get_specialization_counts(opcode) + + return [ + ( + f"{label:>12}", + Count(count), + Ratio(count, total), ) - return rows + for label, count in specialization_counts.items() + ] return calc def calc_specialization_success_failure_table(name: str) -> RowCalculator: def calc(stats: Stats) -> Rows: - opcode_stats = stats.get_opcode_stats("opcode") - family_stats = opcode_stats[name] - total_attempts = 0 - for key in ("specialization.success", "specialization.failure"): - total_attempts += family_stats.get(key, 0) - rows: Rows = [] - if total_attempts: - for key in ("specialization.success", "specialization.failure"): - label = key[len("specialization.") :] - label = label[0].upper() + label[1:] - val = family_stats.get(key, 0) - rows.append((label, Count(val), Ratio(val, total_attempts))) - return rows + values = stats.get_opcode_stats( + "opcode" + ).get_specialization_success_failure(name) + total = sum(values.values()) + if total: + return [ + (label.capitalize(), Count(val), Ratio(val, total)) + for label, val in values.items() + ] + else: + return [] return calc def calc_specialization_failure_kind_table(name: str) -> RowCalculator: def calc(stats: Stats) -> Rows: - def kind_to_text(kind: int, defines: Defines, opname: str): - if kind <= 8: - return pretty(defines[kind][0]) - if opname == "LOAD_SUPER_ATTR": - opname = "SUPER" - elif opname.endswith("ATTR"): - opname = "ATTR" - elif opname in ("FOR_ITER", "SEND"): - opname = "ITER" - elif opname.endswith("SUBSCR"): - opname = "SUBSCR" - for name in defines[kind]: - if name.startswith(opname): - return pretty(name[len(opname) + 1 :]) - return "kind " + str(kind) - - defines = stats.defines opcode_stats = stats.get_opcode_stats("opcode") - family_stats = opcode_stats[name] - total_failures = family_stats.get("specialization.failure", 0) - failure_kinds = [0] * 40 - for key in family_stats: - if not key.startswith("specialization.failure_kind"): - continue - index = int(key[:-1].split("[")[1]) - failure_kinds[index] = family_stats[key] - failures = [(value, index) for (index, value) in enumerate(failure_kinds)] - failures.sort(reverse=True) - rows: Rows = [] - for value, index in failures: - if not value: - continue - rows.append( - ( - kind_to_text(index, defines, name), - Count(value), - Ratio(value, total_failures), - ) - ) - return rows + failures = opcode_stats.get_specialization_failure_kinds(name) + total = opcode_stats.get_specialization_failure_total(name) + + return sorted( + [ + (label, Count(value), Ratio(value, total)) + for label, value in failures.items() + if value + ], + key=itemgetter(1), + reverse=True, + ) return calc def iter_specialization_tables(base_stats: Stats, head_stats: Stats | None = None): opcode_base_stats = base_stats.get_opcode_stats("opcode") - names = opcode_base_stats.keys() + names = opcode_base_stats.get_opcode_names() if head_stats is not None: opcode_head_stats = head_stats.get_opcode_stats("opcode") - names &= opcode_head_stats.keys() # type: ignore + names &= opcode_head_stats.get_opcode_names() # type: ignore else: - opcode_head_stats = {} + opcode_head_stats = None - for name in sorted(names): - if "specializable" not in opcode_base_stats.get(name, {}): + for opcode in sorted(names): + if not opcode_base_stats.is_specializable(opcode): continue - total = sum( - stats.get(name, {}).get(kind, 0) - for kind in TOTAL - for stats in (opcode_base_stats, opcode_head_stats) - ) - if total == 0: + if opcode_base_stats.get_specialization_total(opcode) == 0 and ( + opcode_head_stats is None + or opcode_head_stats.get_specialization_total(opcode) == 0 + ): continue yield Section( - name, - f"specialization stats for {name} family", + opcode, + f"specialization stats for {opcode} family", [ Table( ("Kind", "Count:", "Ratio:"), - calc_specialization_table(name), + calc_specialization_table(opcode), JoinMode.CHANGE, ), Table( ("", "Count:", "Ratio:"), - calc_specialization_success_failure_table(name), + calc_specialization_success_failure_table(opcode), JoinMode.CHANGE, ), Table( ("Failure kind", "Count:", "Ratio:"), - calc_specialization_failure_kind_table(name), + calc_specialization_failure_kind_table(opcode), JoinMode.CHANGE, ), ], @@ -630,24 +794,13 @@ def iter_specialization_tables(base_stats: Stats, head_stats: Stats | None = Non def specialization_effectiveness_section() -> Section: def calc_specialization_effectiveness_table(stats: Stats) -> Rows: opcode_stats = stats.get_opcode_stats("opcode") - total = stats.get_total("opcode") - specialized_instructions = stats.specialized_instructions + total = opcode_stats.get_total_execution_count() - basic = 0 - specialized = 0 - not_specialized = 0 - for name, opcode_stat in opcode_stats.items(): - if "execution_count" not in opcode_stat: - continue - count = opcode_stat["execution_count"] - if "specializable" in opcode_stat: - not_specialized += count - elif name in specialized_instructions: - miss = opcode_stat.get("specialization.miss", 0) - not_specialized += miss - specialized += count - miss - else: - basic += count + ( + basic, + specialized, + not_specialized, + ) = opcode_stats.get_specialized_total_counts() return [ ("Basic", Count(basic), Ratio(basic, total)), @@ -661,105 +814,75 @@ def calc_specialization_effectiveness_table(stats: Stats) -> Rows: def calc_deferred_by_table(stats: Stats) -> Rows: opcode_stats = stats.get_opcode_stats("opcode") - - total = 0 - counts = [] - for name, opcode_stat in opcode_stats.items(): - value = opcode_stat.get("specialization.deferred", 0) - counts.append((value, name)) - total += value - counts.sort(reverse=True) - if total: - return [ - (name, Count(count), Ratio(count, total)) - for (count, name) in counts[:10] - ] - else: + deferred_counts = opcode_stats.get_deferred_counts() + total = sum(deferred_counts.values()) + if total == 0: return [] + return [ + (name, Count(value), Ratio(value, total)) + for name, value in sorted( + deferred_counts.items(), key=itemgetter(1), reverse=True + )[:10] + ] + def calc_misses_by_table(stats: Stats) -> Rows: opcode_stats = stats.get_opcode_stats("opcode") - - total = 0 - counts = [] - for name, opcode_stat in opcode_stats.items(): - # Avoid double counting misses - if "specializable" in opcode_stat: - continue - value = opcode_stat.get("specialization.misses", 0) - counts.append((value, name)) - total += value - counts.sort(reverse=True) - if total: - return [ - (name, Count(count), Ratio(count, total)) - for (count, name) in counts[:10] - ] - else: + misses_counts = opcode_stats.get_misses_counts() + total = sum(misses_counts.values()) + if total == 0: return [] - def iter_specialization_effectiveness_tables( - base_stats: Stats, head_stats: Stats | None = None - ): - yield Table( - ("Instructions", "Count:", "Ratio:"), - calc_specialization_effectiveness_table, - JoinMode.CHANGE, - ) - yield Section( - "Deferred by instruction", - "", - [ - Table( - ("Name", "Count:", "Ratio:"), - calc_deferred_by_table, - JoinMode.CHANGE, - ) - ], - ) - yield Section( - "Misses by instruction", - "", - [ - Table( - ("Name", "Count:", "Ratio:"), - calc_misses_by_table, - JoinMode.CHANGE, - ) - ], - ) + return [ + (name, Count(value), Ratio(value, total)) + for name, value in sorted( + misses_counts.items(), key=itemgetter(1), reverse=True + )[:10] + ] return Section( "Specialization effectiveness", "", - iter_specialization_effectiveness_tables, + [ + Table( + ("Instructions", "Count:", "Ratio:"), + calc_specialization_effectiveness_table, + JoinMode.CHANGE, + ), + Section( + "Deferred by instruction", + "", + [ + Table( + ("Name", "Count:", "Ratio:"), + calc_deferred_by_table, + JoinMode.CHANGE, + ) + ], + ), + Section( + "Misses by instruction", + "", + [ + Table( + ("Name", "Count:", "Ratio:"), + calc_misses_by_table, + JoinMode.CHANGE, + ) + ], + ), + ], ) def call_stats_section() -> Section: def calc_call_stats_table(stats: Stats) -> Rows: - defines = stats.pystats_defines - - total = 0 - for key, value in stats.items(): - if "Calls to" in key: - total += value - - rows: Rows = [] - for key, value in sorted(stats.items()): - if "Calls to" in key: - rows.append((key, Count(value), Ratio(value, total))) - elif key.startswith("Calls "): - name, index = key[:-1].split("[") - index = int(index) - label = f"{name} ({pretty(defines[index][0])})" - rows.append((label, Count(value), Ratio(value, total))) - - for key, value in sorted(stats.items()): - if key.startswith("Frame"): - rows.append((key, Count(value), Ratio(value, total))) - - return rows + call_stats = stats.get_call_stats() + total = sum(v for k, v in call_stats.items() if "Calls to" in k) + return [ + (key, Count(value), Ratio(value, total)) + for key, value in call_stats.items() + ] return Section( "Call stats", @@ -776,33 +899,11 @@ def calc_call_stats_table(stats: Stats) -> Rows: def object_stats_section() -> Section: def calc_object_stats_table(stats: Stats) -> Rows: - total_materializations = stats.get("Object new values", 0) - total_allocations = stats.get("Object allocations", 0) + stats.get( - "Object allocations from freelist", 0 - ) - total_increfs = stats.get("Object interpreter increfs", 0) + stats.get( - "Object increfs", 0 - ) - total_decrefs = stats.get("Object interpreter decrefs", 0) + stats.get( - "Object decrefs", 0 - ) - rows: Rows = [] - for key, value in stats.items(): - if key.startswith("Object"): - if "materialize" in key: - ratio = Ratio(value, total_materializations) - elif "allocations" in key: - ratio = Ratio(value, total_allocations) - elif "increfs" in key: - ratio = Ratio(value, total_increfs) - elif "decrefs" in key: - ratio = Ratio(value, total_decrefs) - else: - ratio = None - label = key[6:].strip() - label = label[0].upper() + label[1:] - rows.append((label, Count(value), ratio)) - return rows + object_stats = stats.get_object_stats() + return [ + (label, Count(value), Ratio(value, den)) + for label, (value, den) in object_stats.items() + ] return Section( "Object stats", @@ -819,16 +920,8 @@ def calc_object_stats_table(stats: Stats) -> Rows: def gc_stats_section() -> Section: def calc_gc_stats(stats: Stats) -> Rows: - gc_stats: list[dict[str, int]] = [] - for key, value in stats.items(): - if not key.startswith("GC"): - continue - n, _, rest = key[3:].partition("]") - name = rest.strip() - gen_n = int(n) - while len(gc_stats) <= gen_n: - gc_stats.append({}) - gc_stats[gen_n][name] = value + gc_stats = stats.get_gc_stats() + return [ ( Count(i), @@ -853,71 +946,30 @@ def calc_gc_stats(stats: Stats) -> Rows: def optimization_section() -> Section: def calc_optimization_table(stats: Stats) -> Rows: - if "Optimization attempts" not in stats: - return [] - - attempts = stats["Optimization attempts"] - created = stats["Optimization traces created"] - executed = stats["Optimization traces executed"] - uops = stats["Optimization uops executed"] - trace_stack_overflow = stats["Optimization trace stack overflow"] - trace_stack_underflow = stats["Optimization trace stack underflow"] - trace_too_long = stats["Optimization trace too long"] - trace_too_short = stats["Optimization trace too short"] - inner_loop = stats["Optimization inner loop"] - recursive_call = stats["Optimization recursive call"] + optimization_stats = stats.get_optimization_stats() return [ - ("Optimization attempts", Count(attempts), ""), - ("Traces created", Count(created), Ratio(created, attempts)), - ("Traces executed", Count(executed), ""), - ("Uops executed", Count(uops), Ratio(uops, executed, percentage=False)), - ( - "Trace stack overflow", - Count(trace_stack_overflow), - Ratio(trace_stack_overflow, created), - ), - ( - "Trace stack underflow", - Count(trace_stack_underflow), - Ratio(trace_stack_underflow, created), - ), - ( - "Trace too long", - Count(trace_too_long), - Ratio(trace_too_long, created), - ), - ( - "Trace too short", - Count(trace_too_short), - Ratio(trace_too_short, created), - ), - ("Inner loop found", Count(inner_loop), Ratio(inner_loop, created)), - ( - "Recursive call", - Count(recursive_call), - Ratio(recursive_call, created), - ), + (label, Count(value), Ratio(value, den, percentage=label != "Uops executed")) + for label, (value, den) in optimization_stats.items() ] def calc_histogram_table(key: str, den: str) -> RowCalculator: def calc(stats: Stats) -> Rows: + histogram = stats.get_histogram(key) + denominator = stats.get(den) + rows: Rows = [] last_non_zero = 0 - for k, v in stats.items(): - if k.startswith(key): - match = re.match(r".+\[([0-9]+)\]", k) - if match is not None: - entry = int(match.groups()[0]) - if v != 0: - last_non_zero = len(rows) - rows.append( - ( - f"<= {entry:,d}", - Count(v), - Ratio(int(v), stats[den]), - ) - ) + for k, v in histogram: + if v != 0: + last_non_zero = len(rows) + rows.append( + ( + f"<= {k:,d}", + Count(v), + Ratio(v, denominator), + ) + ) # Don't include any zero entries at the end rows = rows[: last_non_zero + 1] return rows @@ -926,15 +978,18 @@ def calc(stats: Stats) -> Rows: def calc_unsupported_opcodes_table(stats: Stats) -> Rows: unsupported_opcodes = stats.get_opcode_stats("unsupported_opcode") - data = [] - for opcode, entry in unsupported_opcodes.items(): - data.append((Count(entry["count"]), opcode)) - data.sort(reverse=True) - return [(x[1], x[0]) for x in data] + return sorted( + [ + (opcode, Count(count)) + for opcode, count in unsupported_opcodes.get_opcode_counts().items() + ], + key=itemgetter(1), + reverse=True, + ) def iter_optimization_tables(base_stats: Stats, head_stats: Stats | None = None): - if "Optimization attempts" not in base_stats or ( - head_stats is not None and "Optimization attempts" not in head_stats + if not base_stats.get_optimization_stats() or ( + head_stats is not None and not head_stats.get_optimization_stats() ): return @@ -987,7 +1042,7 @@ def iter_optimization_tables(base_stats: Stats, head_stats: Stats | None = None) def meta_stats_section() -> Section: def calc_rows(stats: Stats) -> Rows: - return [("Number of data files", Count(stats.get("__nfiles__", 0)))] + return [("Number of data files", Count(stats.get("__nfiles__")))] return Section( "Meta stats", @@ -1003,6 +1058,7 @@ def calc_rows(stats: Stats) -> Rows: specialization_section(), specialization_effectiveness_section(), call_stats_section(), + object_stats_section(), gc_stats_section(), optimization_section(), meta_stats_section(), @@ -1080,18 +1136,20 @@ def to_markdown(x): def output_stats(inputs: list[Path], json_output=TextIO | None): match len(inputs): case 1: - stats = Stats(Path(inputs[0])) + data = load_raw_data(Path(inputs[0])) if json_output is not None: - stats.save(json_output) # type: ignore + save_raw_data(data, json_output) # type: ignore + stats = Stats(data) output_markdown(sys.stdout, LAYOUT, stats) case 2: if json_output is not None: raise ValueError( "Can not output to JSON when there are multiple inputs" ) - - base_stats = Stats(Path(inputs[0])) - head_stats = Stats(Path(inputs[1])) + base_data = load_raw_data(Path(inputs[0])) + head_data = load_raw_data(Path(inputs[1])) + base_stats = Stats(base_data) + head_stats = Stats(head_data) output_markdown(sys.stdout, LAYOUT, base_stats, head_stats) From a4fce82339f558c656215b2a6d31c300526b54fa Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Fri, 6 Oct 2023 15:05:15 -0400 Subject: [PATCH 7/8] Typing --- Tools/scripts/summarize_stats.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index 7d60995bee9a26..d3ee778ce459e7 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -399,13 +399,14 @@ def get_optimization_stats(self) -> dict[str, tuple[int, int | None]]: } def get_histogram(self, prefix: str) -> list[tuple[int, int]]: - rows: Rows = [] + rows = [] for k, v in self._data.items(): match = re.match(f"{prefix}\\[([0-9]+)\\]", k) if match is not None: entry = int(match.groups()[0]) rows.append((entry, v)) - return sorted(rows) + rows.sort() + return rows class Count(int): @@ -949,7 +950,11 @@ def calc_optimization_table(stats: Stats) -> Rows: optimization_stats = stats.get_optimization_stats() return [ - (label, Count(value), Ratio(value, den, percentage=label != "Uops executed")) + ( + label, + Count(value), + Ratio(value, den, percentage=label != "Uops executed"), + ) for label, (value, den) in optimization_stats.items() ] From 8a535fe1ddef47e66b7d9383a7a72997241394a1 Mon Sep 17 00:00:00 2001 From: Michael Droettboom Date: Mon, 9 Oct 2023 15:58:21 -0400 Subject: [PATCH 8/8] Fix denominator --- Tools/scripts/summarize_stats.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py index d3ee778ce459e7..071b24a59ef44e 100644 --- a/Tools/scripts/summarize_stats.py +++ b/Tools/scripts/summarize_stats.py @@ -388,14 +388,14 @@ def get_optimization_stats(self) -> dict[str, tuple[int, int | None]]: return { "Optimization attempts": (attempts, None), "Traces created": (created, attempts), + "Trace stack overflow": (trace_stack_overflow, attempts), + "Trace stack underflow": (trace_stack_underflow, attempts), + "Trace too long": (trace_too_long, attempts), + "Trace too short": (trace_too_short, attempts), + "Inner loop found": (inner_loop, attempts), + "Recursive call": (recursive_call, attempts), "Traces executed": (executed, None), "Uops executed": (uops, executed), - "Trace stack overflow": (trace_stack_overflow, created), - "Trace stack underflow": (trace_stack_underflow, created), - "Trace too long": (trace_too_long, created), - "Trace too short": (trace_too_short, created), - "Inner loop found": (inner_loop, created), - "Recursive call": (recursive_call, created), } def get_histogram(self, prefix: str) -> list[tuple[int, int]]: