gh-110019: Refactor summarize_stats (GH-110398)

1 files changed, 1041 insertions, 791 deletions

diff --git a/Tools/scripts/summarize_stats.py b/Tools/scripts/summarize_stats.py
index bdca51d..071b24a 100644
--- a/Tools/scripts/summarize_stats.py
+++ b/Tools/scripts/summarize_stats.py
@@ -2,910 +2,1160 @@
 default stats folders.
 """
 
+from __future__ import annotations
+
 # NOTE: Bytecode introspection modules (opcode, dis, etc.) should only
-# happen when loading a single dataset. When comparing datasets, it
+# be imported when loading a single dataset. When comparing datasets, it
 # could get it wrong, leading to subtle errors.
 
 import argparse
 import collections
-import json
-import os.path
+from collections.abc import KeysView
 from datetime import date
+import enum
+import functools
 import itertools
-import sys
+import json
+from operator import itemgetter
+import os
+from pathlib import Path
 import re
+import sys
+from typing import Any, Callable, TextIO, TypeAlias
+
+
+RawData: TypeAlias = dict[str, Any]
+Rows: TypeAlias = list[tuple]
+Columns: TypeAlias = tuple[str, ...]
+RowCalculator: TypeAlias = Callable[["Stats"], Rows]
+
+
+# TODO: Check for parity
+
 
 if os.name == "nt":
     DEFAULT_DIR = "c:\\temp\\py_stats\\"
 else:
     DEFAULT_DIR = "/tmp/py_stats/"
 
+
+SOURCE_DIR = Path(__file__).parents[2]
+
+
 TOTAL = "specialization.hit", "specialization.miss", "execution_count"
 
 
-def format_ratio(num, den):
-    """
-    Format a ratio as a percentage. When the denominator is 0, returns the empty
-    string.
-    """
-    if den == 0:
-        return ""
-    else:
-        return f"{num/den:.01%}"
+def pretty(name: str) -> str:
+    return name.replace("_", " ").lower()
 
 
-def percentage_to_float(s):
-    """
-    Converts a percentage string to a float.  The empty string is returned as 0.0
-    """
-    if s == "":
-        return 0.0
-    else:
-        assert s[-1] == "%"
-        return float(s[:-1])
+def _load_metadata_from_source():
+    def get_defines(filepath: Path, prefix: str = "SPEC_FAIL"):
+        with open(SOURCE_DIR / filepath) as spec_src:
+            defines = collections.defaultdict(list)
+            start = "#define " + prefix + "_"
+            for line in spec_src:
+                line = line.strip()
+                if not line.startswith(start):
+                    continue
+                line = line[len(start) :]
+                name, val = line.split()
+                defines[int(val.strip())].append(name.strip())
+        return defines
+
+    import opcode
+
+    return {
+        "_specialized_instructions": [
+            op for op in opcode._specialized_opmap.keys() if "__" not in op  # type: ignore
+        ],
+        "_stats_defines": get_defines(
+            Path("Include") / "cpython" / "pystats.h", "EVAL_CALL"
+        ),
+        "_defines": get_defines(Path("Python") / "specialize.c"),
+    }
+
+
+def load_raw_data(input: Path) -> RawData:
+    if input.is_file():
+        with open(input, "r") as fd:
+            data = json.load(fd)
 
+        data["_stats_defines"] = {int(k): v for k, v in data["_stats_defines"].items()}
+        data["_defines"] = {int(k): v for k, v in data["_defines"].items()}
 
-def join_rows(a_rows, b_rows):
-    """
-    Joins two tables together, side-by-side, where the first column in each is a
-    common key.
-    """
-    if len(a_rows) == 0 and len(b_rows) == 0:
-        return []
+        return data
 
-    if len(a_rows):
-        a_ncols = list(set(len(x) for x in a_rows))
-        if len(a_ncols) != 1:
-            raise ValueError("Table a is ragged")
+    elif input.is_dir():
+        stats = collections.Counter[str]()
 
-    if len(b_rows):
-        b_ncols = list(set(len(x) for x in b_rows))
-        if len(b_ncols) != 1:
-            raise ValueError("Table b is ragged")
+        for filename in input.iterdir():
+            with open(filename) as fd:
+                for line in fd:
+                    try:
+                        key, value = line.split(":")
+                    except ValueError:
+                        print(
+                            f"Unparsable line: '{line.strip()}' in {filename}",
+                            file=sys.stderr,
+                        )
+                        continue
+                    stats[key.strip()] += int(value)
+            stats["__nfiles__"] += 1
 
-    if len(a_rows) and len(b_rows) and a_ncols[0] != b_ncols[0]:
-        raise ValueError("Tables have different widths")
+        data = dict(stats)
+        data.update(_load_metadata_from_source())
+        return data
 
-    if len(a_rows):
-        ncols = a_ncols[0]
     else:
-        ncols = b_ncols[0]
+        raise ValueError(f"{input:r} is not a file or directory path")
 
-    default = [""] * (ncols - 1)
-    a_data = {x[0]: x[1:] for x in a_rows}
-    b_data = {x[0]: x[1:] for x in b_rows}
 
-    if len(a_data) != len(a_rows) or len(b_data) != len(b_rows):
-        raise ValueError("Duplicate keys")
+def save_raw_data(data: RawData, json_output: TextIO):
+    json.dump(data, json_output)
 
-    # To preserve ordering, use A's keys as is and then add any in B that aren't
-    # in A
-    keys = list(a_data.keys()) + [k for k in b_data.keys() if k not in a_data]
-    return [(k, *a_data.get(k, default), *b_data.get(k, default)) for k in keys]
 
+class OpcodeStats:
+    """
+    Manages the data related to specific set of opcodes, e.g. tier1 (with prefix
+    "opcode") or tier2 (with prefix "uops").
+    """
 
-def calculate_specialization_stats(family_stats, total):
-    rows = []
-    for key in sorted(family_stats):
-        if key.startswith("specialization.failure_kinds"):
-            continue
-        if key in ("specialization.hit", "specialization.miss"):
-            label = key[len("specialization.") :]
-        elif key == "execution_count":
-            continue
-        elif key in (
-            "specialization.success",
-            "specialization.failure",
-            "specializable",
-        ):
-            continue
-        elif key.startswith("pair"):
-            continue
-        else:
-            label = key
-        rows.append(
-            (
-                f"{label:>12}",
-                f"{family_stats[key]:>12}",
-                format_ratio(family_stats[key], total),
-            )
+    def __init__(self, data: dict[str, Any], defines, specialized_instructions):
+        self._data = data
+        self._defines = defines
+        self._specialized_instructions = specialized_instructions
+
+    def get_opcode_names(self) -> KeysView[str]:
+        return self._data.keys()
+
+    def get_pair_counts(self) -> dict[tuple[str, str], int]:
+        pair_counts = {}
+        for name_i, opcode_stat in self._data.items():
+            for key, value in opcode_stat.items():
+                if value and key.startswith("pair_count"):
+                    name_j, _, _ = key[len("pair_count") + 1 :].partition("]")
+                    pair_counts[(name_i, name_j)] = value
+        return pair_counts
+
+    def get_total_execution_count(self) -> int:
+        return sum(x.get("execution_count", 0) for x in self._data.values())
+
+    def get_execution_counts(self) -> dict[str, tuple[int, int]]:
+        counts = {}
+        for name, opcode_stat in self._data.items():
+            if "execution_count" in opcode_stat:
+                count = opcode_stat["execution_count"]
+                miss = 0
+                if "specializable" not in opcode_stat:
+                    miss = opcode_stat.get("specialization.miss", 0)
+                counts[name] = (count, miss)
+        return counts
+
+    @functools.cache
+    def _get_pred_succ(
+        self,
+    ) -> tuple[dict[str, collections.Counter], dict[str, collections.Counter]]:
+        pair_counts = self.get_pair_counts()
+
+        predecessors: dict[str, collections.Counter] = collections.defaultdict(
+            collections.Counter
         )
-    return rows
+        successors: dict[str, collections.Counter] = collections.defaultdict(
+            collections.Counter
+        )
+        for (first, second), count in pair_counts.items():
+            if count:
+                predecessors[second][first] = count
+                successors[first][second] = count
+
+        return predecessors, successors
+
+    def get_predecessors(self, opcode: str) -> collections.Counter[str]:
+        return self._get_pred_succ()[0][opcode]
+
+    def get_successors(self, opcode: str) -> collections.Counter[str]:
+        return self._get_pred_succ()[1][opcode]
+
+    def _get_stats_for_opcode(self, opcode: str) -> dict[str, int]:
+        return self._data[opcode]
+
+    def get_specialization_total(self, opcode: str) -> int:
+        family_stats = self._get_stats_for_opcode(opcode)
+        return sum(family_stats.get(kind, 0) for kind in TOTAL)
+
+    def get_specialization_counts(self, opcode: str) -> dict[str, int]:
+        family_stats = self._get_stats_for_opcode(opcode)
 
+        result = {}
+        for key, value in sorted(family_stats.items()):
+            if key.startswith("specialization."):
+                label = key[len("specialization.") :]
+                if label in ("success", "failure") or label.startswith("failure_kinds"):
+                    continue
+            elif key in (
+                "execution_count",
+                "specializable",
+            ) or key.startswith("pair"):
+                continue
+            else:
+                label = key
+            result[label] = value
+
+        return result
 
-def calculate_specialization_success_failure(family_stats):
-    total_attempts = 0
-    for key in ("specialization.success", "specialization.failure"):
-        total_attempts += family_stats.get(key, 0)
-    rows = []
-    if total_attempts:
+    def get_specialization_success_failure(self, opcode: str) -> dict[str, int]:
+        family_stats = self._get_stats_for_opcode(opcode)
+        result = {}
         for key in ("specialization.success", "specialization.failure"):
             label = key[len("specialization.") :]
-            label = label[0].upper() + label[1:]
             val = family_stats.get(key, 0)
-            rows.append((label, val, format_ratio(val, total_attempts)))
-    return rows
-
-
-def calculate_specialization_failure_kinds(name, family_stats, defines):
-    total_failures = family_stats.get("specialization.failure", 0)
-    failure_kinds = [0] * 40
-    for key in family_stats:
-        if not key.startswith("specialization.failure_kind"):
-            continue
-        _, index = key[:-1].split("[")
-        index = int(index)
-        failure_kinds[index] = family_stats[key]
-    failures = [(value, index) for (index, value) in enumerate(failure_kinds)]
-    failures.sort(reverse=True)
-    rows = []
-    for value, index in failures:
-        if not value:
-            continue
-        rows.append(
-            (
-                kind_to_text(index, defines, name),
-                value,
-                format_ratio(value, total_failures),
-            )
-        )
-    return rows
-
-
-def print_specialization_stats(name, family_stats, defines):
-    if "specializable" not in family_stats:
-        return
-    total = sum(family_stats.get(kind, 0) for kind in TOTAL)
-    if total == 0:
-        return
-    with Section(name, 3, f"specialization stats for {name} family"):
-        rows = calculate_specialization_stats(family_stats, total)
-        emit_table(("Kind", "Count", "Ratio"), rows)
-        rows = calculate_specialization_success_failure(family_stats)
-        if rows:
-            print_title("Specialization attempts", 4)
-            emit_table(("", "Count:", "Ratio:"), rows)
-            rows = calculate_specialization_failure_kinds(name, family_stats, defines)
-            emit_table(("Failure kind", "Count:", "Ratio:"), rows)
-
-
-def print_comparative_specialization_stats(
-    name, base_family_stats, head_family_stats, defines
-):
-    if "specializable" not in base_family_stats:
-        return
-
-    base_total = sum(base_family_stats.get(kind, 0) for kind in TOTAL)
-    head_total = sum(head_family_stats.get(kind, 0) for kind in TOTAL)
-    if base_total + head_total == 0:
-        return
-    with Section(name, 3, f"specialization stats for {name} family"):
-        base_rows = calculate_specialization_stats(base_family_stats, base_total)
-        head_rows = calculate_specialization_stats(head_family_stats, head_total)
-        emit_table(
-            ("Kind", "Base Count", "Base Ratio", "Head Count", "Head Ratio"),
-            join_rows(base_rows, head_rows),
-        )
-        base_rows = calculate_specialization_success_failure(base_family_stats)
-        head_rows = calculate_specialization_success_failure(head_family_stats)
-        rows = join_rows(base_rows, head_rows)
-        if rows:
-            print_title("Specialization attempts", 4)
-            emit_table(
-                ("", "Base Count:", "Base Ratio:", "Head Count:", "Head Ratio:"), rows
-            )
-            base_rows = calculate_specialization_failure_kinds(
-                name, base_family_stats, defines
-            )
-            head_rows = calculate_specialization_failure_kinds(
-                name, head_family_stats, defines
-            )
-            emit_table(
-                (
-                    "Failure kind",
-                    "Base Count:",
-                    "Base Ratio:",
-                    "Head Count:",
-                    "Head Ratio:",
-                ),
-                join_rows(base_rows, head_rows),
-            )
+            result[label] = val
+        return result
+
+    def get_specialization_failure_total(self, opcode: str) -> int:
+        return self._get_stats_for_opcode(opcode).get("specialization.failure", 0)
+
+    def get_specialization_failure_kinds(self, opcode: str) -> dict[str, int]:
+        def kind_to_text(kind: int, opcode: str):
+            if kind <= 8:
+                return pretty(self._defines[kind][0])
+            if opcode == "LOAD_SUPER_ATTR":
+                opcode = "SUPER"
+            elif opcode.endswith("ATTR"):
+                opcode = "ATTR"
+            elif opcode in ("FOR_ITER", "SEND"):
+                opcode = "ITER"
+            elif opcode.endswith("SUBSCR"):
+                opcode = "SUBSCR"
+            for name in self._defines[kind]:
+                if name.startswith(opcode):
+                    return pretty(name[len(opcode) + 1 :])
+            return "kind " + str(kind)
+
+        family_stats = self._get_stats_for_opcode(opcode)
+        failure_kinds = [0] * 40
+        for key in family_stats:
+            if not key.startswith("specialization.failure_kind"):
+                continue
+            index = int(key[:-1].split("[")[1])
+            failure_kinds[index] = family_stats[key]
+        return {
+            kind_to_text(index, opcode): value
+            for (index, value) in enumerate(failure_kinds)
+            if value
+        }
 
+    def is_specializable(self, opcode: str) -> bool:
+        return "specializable" in self._get_stats_for_opcode(opcode)
 
-def gather_stats(input):
-    # Note the output of this function must be JSON-serializable
+    def get_specialized_total_counts(self) -> tuple[int, int, int]:
+        basic = 0
+        specialized = 0
+        not_specialized = 0
+        for opcode, opcode_stat in self._data.items():
+            if "execution_count" not in opcode_stat:
+                continue
+            count = opcode_stat["execution_count"]
+            if "specializable" in opcode_stat:
+                not_specialized += count
+            elif opcode in self._specialized_instructions:
+                miss = opcode_stat.get("specialization.miss", 0)
+                not_specialized += miss
+                specialized += count - miss
+            else:
+                basic += count
+        return basic, specialized, not_specialized
 
-    if os.path.isfile(input):
-        with open(input, "r") as fd:
-            stats = json.load(fd)
+    def get_deferred_counts(self) -> dict[str, int]:
+        return {
+            opcode: opcode_stat.get("specialization.deferred", 0)
+            for opcode, opcode_stat in self._data.items()
+        }
 
-        stats["_stats_defines"] = {
-            int(k): v for k, v in stats["_stats_defines"].items()
+    def get_misses_counts(self) -> dict[str, int]:
+        return {
+            opcode: opcode_stat.get("specialization.miss", 0)
+            for opcode, opcode_stat in self._data.items()
+            if not self.is_specializable(opcode)
         }
-        stats["_defines"] = {int(k): v for k, v in stats["_defines"].items()}
-        return stats
 
-    elif os.path.isdir(input):
-        stats = collections.Counter()
-        for filename in os.listdir(input):
-            with open(os.path.join(input, filename)) as fd:
-                for line in fd:
-                    try:
-                        key, value = line.split(":")
-                    except ValueError:
-                        print(
-                            f"Unparsable line: '{line.strip()}' in  {filename}",
-                            file=sys.stderr,
-                        )
-                        continue
-                    key = key.strip()
-                    value = int(value)
-                    stats[key] += value
-            stats["__nfiles__"] += 1
+    def get_opcode_counts(self) -> dict[str, int]:
+        counts = {}
+        for opcode, entry in self._data.items():
+            count = entry.get("count", 0)
+            if count:
+                counts[opcode] = count
+        return counts
 
-        import opcode
 
-        stats["_specialized_instructions"] = [
-            op for op in opcode._specialized_opmap.keys() if "__" not in op
-        ]
-        stats["_stats_defines"] = get_stats_defines()
-        stats["_defines"] = get_defines()
+class Stats:
+    def __init__(self, data: RawData):
+        self._data = data
 
-        return stats
-    else:
-        raise ValueError(f"{input:r} is not a file or directory path")
+    def get(self, key: str) -> int:
+        return self._data.get(key, 0)
 
+    @functools.cache
+    def get_opcode_stats(self, prefix: str) -> OpcodeStats:
+        opcode_stats = collections.defaultdict[str, dict](dict)
+        for key, value in self._data.items():
+            if not key.startswith(prefix):
+                continue
+            name, _, rest = key[len(prefix) + 1 :].partition("]")
+            opcode_stats[name][rest.strip(".")] = value
+        return OpcodeStats(
+            opcode_stats,
+            self._data["_defines"],
+            self._data["_specialized_instructions"],
+        )
 
-def extract_opcode_stats(stats, prefix):
-    opcode_stats = collections.defaultdict(dict)
-    for key, value in stats.items():
-        if not key.startswith(prefix):
-            continue
-        name, _, rest = key[len(prefix) + 1 :].partition("]")
-        opcode_stats[name][rest.strip(".")] = value
-    return opcode_stats
-
-
-def parse_kinds(spec_src, prefix="SPEC_FAIL"):
-    defines = collections.defaultdict(list)
-    start = "#define " + prefix + "_"
-    for line in spec_src:
-        line = line.strip()
-        if not line.startswith(start):
-            continue
-        line = line[len(start) :]
-        name, val = line.split()
-        defines[int(val.strip())].append(name.strip())
-    return defines
-
-
-def pretty(defname):
-    return defname.replace("_", " ").lower()
-
-
-def kind_to_text(kind, defines, opname):
-    if kind <= 8:
-        return pretty(defines[kind][0])
-    if opname == "LOAD_SUPER_ATTR":
-        opname = "SUPER"
-    elif opname.endswith("ATTR"):
-        opname = "ATTR"
-    elif opname in ("FOR_ITER", "SEND"):
-        opname = "ITER"
-    elif opname.endswith("SUBSCR"):
-        opname = "SUBSCR"
-    for name in defines[kind]:
-        if name.startswith(opname):
-            return pretty(name[len(opname) + 1 :])
-    return "kind " + str(kind)
-
-
-def categorized_counts(opcode_stats, specialized_instructions):
-    basic = 0
-    specialized = 0
-    not_specialized = 0
-    for name, opcode_stat in opcode_stats.items():
-        if "execution_count" not in opcode_stat:
-            continue
-        count = opcode_stat["execution_count"]
-        if "specializable" in opcode_stat:
-            not_specialized += count
-        elif name in specialized_instructions:
-            miss = opcode_stat.get("specialization.miss", 0)
-            not_specialized += miss
-            specialized += count - miss
+    def get_call_stats(self) -> dict[str, int]:
+        defines = self._data["_stats_defines"]
+        result = {}
+        for key, value in sorted(self._data.items()):
+            if "Calls to" in key:
+                result[key] = value
+            elif key.startswith("Calls "):
+                name, index = key[:-1].split("[")
+                label = f"{name} ({pretty(defines[int(index)][0])})"
+                result[label] = value
+
+        for key, value in sorted(self._data.items()):
+            if key.startswith("Frame"):
+                result[key] = value
+
+        return result
+
+    def get_object_stats(self) -> dict[str, tuple[int, int]]:
+        total_materializations = self._data.get("Object new values", 0)
+        total_allocations = self._data.get("Object allocations", 0) + self._data.get(
+            "Object allocations from freelist", 0
+        )
+        total_increfs = self._data.get(
+            "Object interpreter increfs", 0
+        ) + self._data.get("Object increfs", 0)
+        total_decrefs = self._data.get(
+            "Object interpreter decrefs", 0
+        ) + self._data.get("Object decrefs", 0)
+
+        result = {}
+        for key, value in self._data.items():
+            if key.startswith("Object"):
+                if "materialize" in key:
+                    den = total_materializations
+                elif "allocations" in key:
+                    den = total_allocations
+                elif "increfs" in key:
+                    den = total_increfs
+                elif "decrefs" in key:
+                    den = total_decrefs
+                else:
+                    den = None
+                label = key[6:].strip()
+                label = label[0].upper() + label[1:]
+                result[label] = (value, den)
+        return result
+
+    def get_gc_stats(self) -> list[dict[str, int]]:
+        gc_stats: list[dict[str, int]] = []
+        for key, value in self._data.items():
+            if not key.startswith("GC"):
+                continue
+            n, _, rest = key[3:].partition("]")
+            name = rest.strip()
+            gen_n = int(n)
+            while len(gc_stats) <= gen_n:
+                gc_stats.append({})
+            gc_stats[gen_n][name] = value
+        return gc_stats
+
+    def get_optimization_stats(self) -> dict[str, tuple[int, int | None]]:
+        if "Optimization attempts" not in self._data:
+            return {}
+
+        attempts = self._data["Optimization attempts"]
+        created = self._data["Optimization traces created"]
+        executed = self._data["Optimization traces executed"]
+        uops = self._data["Optimization uops executed"]
+        trace_stack_overflow = self._data["Optimization trace stack overflow"]
+        trace_stack_underflow = self._data["Optimization trace stack underflow"]
+        trace_too_long = self._data["Optimization trace too long"]
+        trace_too_short = self._data["Optimization trace too short"]
+        inner_loop = self._data["Optimization inner loop"]
+        recursive_call = self._data["Optimization recursive call"]
+
+        return {
+            "Optimization attempts": (attempts, None),
+            "Traces created": (created, attempts),
+            "Trace stack overflow": (trace_stack_overflow, attempts),
+            "Trace stack underflow": (trace_stack_underflow, attempts),
+            "Trace too long": (trace_too_long, attempts),
+            "Trace too short": (trace_too_short, attempts),
+            "Inner loop found": (inner_loop, attempts),
+            "Recursive call": (recursive_call, attempts),
+            "Traces executed": (executed, None),
+            "Uops executed": (uops, executed),
+        }
+
+    def get_histogram(self, prefix: str) -> list[tuple[int, int]]:
+        rows = []
+        for k, v in self._data.items():
+            match = re.match(f"{prefix}\\[([0-9]+)\\]", k)
+            if match is not None:
+                entry = int(match.groups()[0])
+                rows.append((entry, v))
+        rows.sort()
+        return rows
+
+
+class Count(int):
+    def markdown(self) -> str:
+        return format(self, ",d")
+
+
+class Ratio:
+    def __init__(self, num: int, den: int | None, percentage: bool = True):
+        self.num = num
+        self.den = den
+        self.percentage = percentage
+        if den == 0 and num != 0:
+            raise ValueError("Invalid denominator")
+
+    def __float__(self):
+        if self.den == 0:
+            return 0.0
+        elif self.den is None:
+            return self.num
+        else:
+            return self.num / self.den
+
+    def markdown(self) -> str:
+        if self.den == 0 or self.den is None:
+            return ""
+        elif self.percentage:
+            return f"{self.num / self.den:,.01%}"
+        else:
+            return f"{self.num / self.den:,.02f}"
+
+
+class DiffRatio(Ratio):
+    def __init__(self, base: int | str, head: int | str):
+        if isinstance(base, str) or isinstance(head, str):
+            super().__init__(0, 0)
         else:
-            basic += count
-    return basic, not_specialized, specialized
+            super().__init__(head - base, base)
+
+
+class JoinMode(enum.Enum):
+    # Join using the first column as a key
+    SIMPLE = 0
+    # Join using the first column as a key, and indicate the change in the
+    # second column of each input table as a new column
+    CHANGE = 1
+    # Join using the first column as a key, indicating the change in the second
+    # column of each input table as a ne column, and omit all other columns
+    CHANGE_ONE_COLUMN = 2
+
+
+class Table:
+    """
+    A Table defines how to convert a set of Stats into a specific set of rows
+    displaying some aspect of the data.
+    """
+
+    def __init__(
+        self,
+        column_names: Columns,
+        calc_rows: RowCalculator,
+        join_mode: JoinMode = JoinMode.SIMPLE,
+    ):
+        self.columns = column_names
+        self.calc_rows = calc_rows
+        self.join_mode = join_mode
+
+    def join_row(self, key: str, row_a: tuple, row_b: tuple) -> tuple:
+        match self.join_mode:
+            case JoinMode.SIMPLE:
+                return (key, *row_a, *row_b)
+            case JoinMode.CHANGE:
+                return (key, *row_a, *row_b, DiffRatio(row_a[0], row_b[0]))
+            case JoinMode.CHANGE_ONE_COLUMN:
+                return (key, row_a[0], row_b[0], DiffRatio(row_a[0], row_b[0]))
+
+    def join_columns(self, columns: Columns) -> Columns:
+        match self.join_mode:
+            case JoinMode.SIMPLE:
+                return (
+                    columns[0],
+                    *("Base " + x for x in columns[1:]),
+                    *("Head " + x for x in columns[1:]),
+                )
+            case JoinMode.CHANGE:
+                return (
+                    columns[0],
+                    *("Base " + x for x in columns[1:]),
+                    *("Head " + x for x in columns[1:]),
+                ) + ("Change:",)
+            case JoinMode.CHANGE_ONE_COLUMN:
+                return (
+                    columns[0],
+                    "Base " + columns[1],
+                    "Head " + columns[1],
+                    "Change:",
+                )
+
+    def join_tables(self, rows_a: Rows, rows_b: Rows) -> tuple[Columns, Rows]:
+        ncols = len(self.columns)
+
+        default = ("",) * (ncols - 1)
+        data_a = {x[0]: x[1:] for x in rows_a}
+        data_b = {x[0]: x[1:] for x in rows_b}
 
+        if len(data_a) != len(rows_a) or len(data_b) != len(rows_b):
+            raise ValueError("Duplicate keys")
 
-def print_title(name, level=2):
-    print("#" * level, name)
-    print()
+        # To preserve ordering, use A's keys as is and then add any in B that
+        # aren't in A
+        keys = list(data_a.keys()) + [k for k in data_b.keys() if k not in data_a]
+        rows = [
+            self.join_row(k, data_a.get(k, default), data_b.get(k, default))
+            for k in keys
+        ]
+        if self.join_mode in (JoinMode.CHANGE, JoinMode.CHANGE_ONE_COLUMN):
+            rows.sort(key=lambda row: abs(float(row[-1])), reverse=True)
+
+        columns = self.join_columns(self.columns)
+        return columns, rows
+
+    def get_table(
+        self, base_stats: Stats, head_stats: Stats | None = None
+    ) -> tuple[Columns, Rows]:
+        if head_stats is None:
+            rows = self.calc_rows(base_stats)
+            return self.columns, rows
+        else:
+            rows_a = self.calc_rows(base_stats)
+            rows_b = self.calc_rows(head_stats)
+            cols, rows = self.join_tables(rows_a, rows_b)
+            return cols, rows
 
 
 class Section:
-    def __init__(self, title, level=2, summary=None):
+    """
+    A Section defines a section of the output document.
+    """
+
+    def __init__(
+        self,
+        title: str = "",
+        summary: str = "",
+        part_iter=None,
+        comparative: bool = True,
+    ):
         self.title = title
-        self.level = level
-        if summary is None:
+        if not summary:
             self.summary = title.lower()
         else:
             self.summary = summary
+        if part_iter is None:
+            part_iter = []
+        if isinstance(part_iter, list):
 
-    def __enter__(self):
-        print_title(self.title, self.level)
-        print("<details>")
-        print("<summary>", self.summary, "</summary>")
-        print()
-        return self
+            def iter_parts(base_stats: Stats, head_stats: Stats | None):
+                yield from part_iter
 
-    def __exit__(*args):
-        print()
-        print("</details>")
-        print()
+            self.part_iter = iter_parts
+        else:
+            self.part_iter = part_iter
+        self.comparative = comparative
 
 
-def to_str(x):
-    if isinstance(x, int):
-        return format(x, ",d")
-    else:
-        return str(x)
-
-
-def emit_table(header, rows):
-    width = len(header)
-    header_line = "|"
-    under_line = "|"
-    for item in header:
-        under = "---"
-        if item.endswith(":"):
-            item = item[:-1]
-            under += ":"
-        header_line += item + " | "
-        under_line += under + "|"
-    print(header_line)
-    print(under_line)
-    for row in rows:
-        if width is not None and len(row) != width:
-            raise ValueError("Wrong number of elements in row '" + str(row) + "'")
-        print("|", " | ".join(to_str(i) for i in row), "|")
-    print()
-
-
-def emit_histogram(title, stats, key, total):
-    rows = []
-    for k, v in stats.items():
-        if k.startswith(key):
-            entry = int(re.match(r".+\[([0-9]+)\]", k).groups()[0])
-            rows.append((f"<= {entry}", int(v), format_ratio(int(v), total)))
-    # Don't include larger buckets with 0 entries
-    for j in range(len(rows) - 1, -1, -1):
-        if rows[j][1] != 0:
-            break
-    rows = rows[: j + 1]
-
-    print(f"**{title}**\n")
-    emit_table(("Range", "Count:", "Ratio:"), rows)
-
-
-def calculate_execution_counts(opcode_stats, total):
-    counts = []
-    for name, opcode_stat in opcode_stats.items():
-        if "execution_count" in opcode_stat:
-            count = opcode_stat["execution_count"]
-            miss = 0
-            if "specializable" not in opcode_stat:
-                miss = opcode_stat.get("specialization.miss")
-            counts.append((count, name, miss))
-    counts.sort(reverse=True)
-    cumulative = 0
-    rows = []
-    for count, name, miss in counts:
-        cumulative += count
-        if miss:
-            miss = format_ratio(miss, count)
-        else:
-            miss = ""
-        rows.append(
-            (
-                name,
-                count,
-                format_ratio(count, total),
-                format_ratio(cumulative, total),
-                miss,
+def calc_execution_count_table(prefix: str) -> RowCalculator:
+    def calc(stats: Stats) -> Rows:
+        opcode_stats = stats.get_opcode_stats(prefix)
+        counts = opcode_stats.get_execution_counts()
+        total = opcode_stats.get_total_execution_count()
+        cumulative = 0
+        rows: Rows = []
+        for opcode, (count, miss) in sorted(
+            counts.items(), key=itemgetter(1), reverse=True
+        ):
+            cumulative += count
+            if miss:
+                miss_val = Ratio(miss, count)
+            else:
+                miss_val = None
+            rows.append(
+                (
+                    opcode,
+                    Count(count),
+                    Ratio(count, total),
+                    Ratio(cumulative, total),
+                    miss_val,
+                )
             )
-        )
-    return rows
+        return rows
 
+    return calc
 
-def emit_execution_counts(opcode_stats, total):
-    with Section("Execution counts", summary="execution counts for all instructions"):
-        rows = calculate_execution_counts(opcode_stats, total)
-        emit_table(("Name", "Count:", "Self:", "Cumulative:", "Miss ratio:"), rows)
 
+def execution_count_section() -> Section:
+    return Section(
+        "Execution counts",
+        "execution counts for all instructions",
+        [
+            Table(
+                ("Name", "Count:", "Self:", "Cumulative:", "Miss ratio:"),
+                calc_execution_count_table("opcode"),
+                join_mode=JoinMode.CHANGE_ONE_COLUMN,
+            )
+        ],
+    )
 
-def _emit_comparative_execution_counts(base_rows, head_rows):
-    base_data = {x[0]: x[1:] for x in base_rows}
-    head_data = {x[0]: x[1:] for x in head_rows}
-    opcodes = base_data.keys() | head_data.keys()
 
-    rows = []
-    default = [0, "0.0%", "0.0%", 0]
-    for opcode in opcodes:
-        base_entry = base_data.get(opcode, default)
-        head_entry = head_data.get(opcode, default)
-        if base_entry[0] == 0:
-            change = 1
-        else:
-            change = (head_entry[0] - base_entry[0]) / base_entry[0]
-        rows.append((opcode, base_entry[0], head_entry[0], f"{change:0.1%}"))
+def pair_count_section() -> Section:
+    def calc_pair_count_table(stats: Stats) -> Rows:
+        opcode_stats = stats.get_opcode_stats("opcode")
+        pair_counts = opcode_stats.get_pair_counts()
+        total = opcode_stats.get_total_execution_count()
 
-    rows.sort(key=lambda x: abs(percentage_to_float(x[-1])), reverse=True)
+        cumulative = 0
+        rows: Rows = []
+        for (opcode_i, opcode_j), count in itertools.islice(
+            sorted(pair_counts.items(), key=itemgetter(1), reverse=True), 100
+        ):
+            cumulative += count
+            rows.append(
+                (
+                    f"{opcode_i} {opcode_j}",
+                    Count(count),
+                    Ratio(count, total),
+                    Ratio(cumulative, total),
+                )
+            )
+        return rows
+
+    return Section(
+        "Pair counts",
+        "Pair counts for top 100 pairs",
+        [
+            Table(
+                ("Pair", "Count:", "Self:", "Cumulative:"),
+                calc_pair_count_table,
+            )
+        ],
+        comparative=False,
+    )
 
-    emit_table(("Name", "Base Count:", "Head Count:", "Change:"), rows)
 
+def pre_succ_pairs_section() -> Section:
+    def iter_pre_succ_pairs_tables(base_stats: Stats, head_stats: Stats | None = None):
+        assert head_stats is None
 
-def emit_comparative_execution_counts(
-    base_opcode_stats, base_total, head_opcode_stats, head_total, level=2
-):
-    with Section(
-        "Execution counts", summary="execution counts for all instructions", level=level
-    ):
-        base_rows = calculate_execution_counts(base_opcode_stats, base_total)
-        head_rows = calculate_execution_counts(head_opcode_stats, head_total)
-        _emit_comparative_execution_counts(base_rows, head_rows)
+        opcode_stats = base_stats.get_opcode_stats("opcode")
 
+        for opcode in opcode_stats.get_opcode_names():
+            predecessors = opcode_stats.get_predecessors(opcode)
+            successors = opcode_stats.get_successors(opcode)
+            predecessors_total = predecessors.total()
+            successors_total = successors.total()
+            if predecessors_total == 0 and successors_total == 0:
+                continue
+            pred_rows = [
+                (pred, Count(count), Ratio(count, predecessors_total))
+                for (pred, count) in predecessors.most_common(5)
+            ]
+            succ_rows = [
+                (succ, Count(count), Ratio(count, successors_total))
+                for (succ, count) in successors.most_common(5)
+            ]
+
+            yield Section(
+                opcode,
+                f"Successors and predecessors for {opcode}",
+                [
+                    Table(
+                        ("Predecessors", "Count:", "Percentage:"),
+                        lambda *_: pred_rows,  # type: ignore
+                    ),
+                    Table(
+                        ("Successors", "Count:", "Percentage:"),
+                        lambda *_: succ_rows,  # type: ignore
+                    ),
+                ],
+            )
 
-def get_defines():
-    spec_path = os.path.join(os.path.dirname(__file__), "../../Python/specialize.c")
-    with open(spec_path) as spec_src:
-        defines = parse_kinds(spec_src)
-    return defines
+    return Section(
+        "Predecessor/Successor Pairs",
+        "Top 5 predecessors and successors of each opcode",
+        iter_pre_succ_pairs_tables,
+        comparative=False,
+    )
 
 
-def emit_specialization_stats(opcode_stats, defines):
-    with Section("Specialization stats", summary="specialization stats by family"):
-        for name, opcode_stat in opcode_stats.items():
-            print_specialization_stats(name, opcode_stat, defines)
+def specialization_section() -> Section:
+    def calc_specialization_table(opcode: str) -> RowCalculator:
+        def calc(stats: Stats) -> Rows:
+            opcode_stats = stats.get_opcode_stats("opcode")
+            total = opcode_stats.get_specialization_total(opcode)
+            specialization_counts = opcode_stats.get_specialization_counts(opcode)
 
+            return [
+                (
+                    f"{label:>12}",
+                    Count(count),
+                    Ratio(count, total),
+                )
+                for label, count in specialization_counts.items()
+            ]
 
-def emit_comparative_specialization_stats(
-    base_opcode_stats, head_opcode_stats, defines
-):
-    with Section("Specialization stats", summary="specialization stats by family"):
-        opcodes = set(base_opcode_stats.keys()) & set(head_opcode_stats.keys())
-        for opcode in opcodes:
-            print_comparative_specialization_stats(
-                opcode, base_opcode_stats[opcode], head_opcode_stats[opcode], defines
+        return calc
+
+    def calc_specialization_success_failure_table(name: str) -> RowCalculator:
+        def calc(stats: Stats) -> Rows:
+            values = stats.get_opcode_stats(
+                "opcode"
+            ).get_specialization_success_failure(name)
+            total = sum(values.values())
+            if total:
+                return [
+                    (label.capitalize(), Count(val), Ratio(val, total))
+                    for label, val in values.items()
+                ]
+            else:
+                return []
+
+        return calc
+
+    def calc_specialization_failure_kind_table(name: str) -> RowCalculator:
+        def calc(stats: Stats) -> Rows:
+            opcode_stats = stats.get_opcode_stats("opcode")
+            failures = opcode_stats.get_specialization_failure_kinds(name)
+            total = opcode_stats.get_specialization_failure_total(name)
+
+            return sorted(
+                [
+                    (label, Count(value), Ratio(value, total))
+                    for label, value in failures.items()
+                    if value
+                ],
+                key=itemgetter(1),
+                reverse=True,
             )
 
+        return calc
+
+    def iter_specialization_tables(base_stats: Stats, head_stats: Stats | None = None):
+        opcode_base_stats = base_stats.get_opcode_stats("opcode")
+        names = opcode_base_stats.get_opcode_names()
+        if head_stats is not None:
+            opcode_head_stats = head_stats.get_opcode_stats("opcode")
+            names &= opcode_head_stats.get_opcode_names()  # type: ignore
+        else:
+            opcode_head_stats = None
 
-def calculate_specialization_effectiveness(
-    opcode_stats, total, specialized_instructions
-):
-    basic, not_specialized, specialized = categorized_counts(
-        opcode_stats, specialized_instructions
+        for opcode in sorted(names):
+            if not opcode_base_stats.is_specializable(opcode):
+                continue
+            if opcode_base_stats.get_specialization_total(opcode) == 0 and (
+                opcode_head_stats is None
+                or opcode_head_stats.get_specialization_total(opcode) == 0
+            ):
+                continue
+            yield Section(
+                opcode,
+                f"specialization stats for {opcode} family",
+                [
+                    Table(
+                        ("Kind", "Count:", "Ratio:"),
+                        calc_specialization_table(opcode),
+                        JoinMode.CHANGE,
+                    ),
+                    Table(
+                        ("", "Count:", "Ratio:"),
+                        calc_specialization_success_failure_table(opcode),
+                        JoinMode.CHANGE,
+                    ),
+                    Table(
+                        ("Failure kind", "Count:", "Ratio:"),
+                        calc_specialization_failure_kind_table(opcode),
+                        JoinMode.CHANGE,
+                    ),
+                ],
+            )
+
+    return Section(
+        "Specialization stats",
+        "specialization stats by family",
+        iter_specialization_tables,
     )
-    return [
-        ("Basic", basic, format_ratio(basic, total)),
-        ("Not specialized", not_specialized, format_ratio(not_specialized, total)),
-        ("Specialized", specialized, format_ratio(specialized, total)),
-    ]
 
 
-def emit_specialization_overview(opcode_stats, total, specialized_instructions):
-    with Section("Specialization effectiveness"):
-        rows = calculate_specialization_effectiveness(
-            opcode_stats, total, specialized_instructions
-        )
-        emit_table(("Instructions", "Count:", "Ratio:"), rows)
-        for title, field in (
-            ("Deferred", "specialization.deferred"),
-            ("Misses", "specialization.miss"),
-        ):
-            total = 0
-            counts = []
-            for name, opcode_stat in opcode_stats.items():
-                # Avoid double counting misses
-                if title == "Misses" and "specializable" in opcode_stat:
-                    continue
-                value = opcode_stat.get(field, 0)
-                counts.append((value, name))
-                total += value
-            counts.sort(reverse=True)
-            if total:
-                with Section(f"{title} by instruction", 3):
-                    rows = [
-                        (name, count, format_ratio(count, total))
-                        for (count, name) in counts[:10]
-                    ]
-                    emit_table(("Name", "Count:", "Ratio:"), rows)
-
-
-def emit_comparative_specialization_overview(
-    base_opcode_stats,
-    base_total,
-    head_opcode_stats,
-    head_total,
-    specialized_instructions,
-):
-    with Section("Specialization effectiveness"):
-        base_rows = calculate_specialization_effectiveness(
-            base_opcode_stats, base_total, specialized_instructions
-        )
-        head_rows = calculate_specialization_effectiveness(
-            head_opcode_stats, head_total, specialized_instructions
-        )
-        emit_table(
+def specialization_effectiveness_section() -> Section:
+    def calc_specialization_effectiveness_table(stats: Stats) -> Rows:
+        opcode_stats = stats.get_opcode_stats("opcode")
+        total = opcode_stats.get_total_execution_count()
+
+        (
+            basic,
+            specialized,
+            not_specialized,
+        ) = opcode_stats.get_specialized_total_counts()
+
+        return [
+            ("Basic", Count(basic), Ratio(basic, total)),
             (
-                "Instructions",
-                "Base Count:",
-                "Base Ratio:",
-                "Head Count:",
-                "Head Ratio:",
+                "Not specialized",
+                Count(not_specialized),
+                Ratio(not_specialized, total),
             ),
-            join_rows(base_rows, head_rows),
-        )
+            ("Specialized", Count(specialized), Ratio(specialized, total)),
+        ]
+
+    def calc_deferred_by_table(stats: Stats) -> Rows:
+        opcode_stats = stats.get_opcode_stats("opcode")
+        deferred_counts = opcode_stats.get_deferred_counts()
+        total = sum(deferred_counts.values())
+        if total == 0:
+            return []
+
+        return [
+            (name, Count(value), Ratio(value, total))
+            for name, value in sorted(
+                deferred_counts.items(), key=itemgetter(1), reverse=True
+            )[:10]
+        ]
 
+    def calc_misses_by_table(stats: Stats) -> Rows:
+        opcode_stats = stats.get_opcode_stats("opcode")
+        misses_counts = opcode_stats.get_misses_counts()
+        total = sum(misses_counts.values())
+        if total == 0:
+            return []
+
+        return [
+            (name, Count(value), Ratio(value, total))
+            for name, value in sorted(
+                misses_counts.items(), key=itemgetter(1), reverse=True
+            )[:10]
+        ]
 
-def get_stats_defines():
-    stats_path = os.path.join(
-        os.path.dirname(__file__), "../../Include/cpython/pystats.h"
+    return Section(
+        "Specialization effectiveness",
+        "",
+        [
+            Table(
+                ("Instructions", "Count:", "Ratio:"),
+                calc_specialization_effectiveness_table,
+                JoinMode.CHANGE,
+            ),
+            Section(
+                "Deferred by instruction",
+                "",
+                [
+                    Table(
+                        ("Name", "Count:", "Ratio:"),
+                        calc_deferred_by_table,
+                        JoinMode.CHANGE,
+                    )
+                ],
+            ),
+            Section(
+                "Misses by instruction",
+                "",
+                [
+                    Table(
+                        ("Name", "Count:", "Ratio:"),
+                        calc_misses_by_table,
+                        JoinMode.CHANGE,
+                    )
+                ],
+            ),
+        ],
     )
-    with open(stats_path) as stats_src:
-        defines = parse_kinds(stats_src, prefix="EVAL_CALL")
-    return defines
-
-
-def calculate_call_stats(stats, defines):
-    total = 0
-    for key, value in stats.items():
-        if "Calls to" in key:
-            total += value
-            rows = []
-    for key, value in stats.items():
-        if "Calls to" in key:
-            rows.append((key, value, format_ratio(value, total)))
-        elif key.startswith("Calls "):
-            name, index = key[:-1].split("[")
-            index = int(index)
-            label = name + " (" + pretty(defines[index][0]) + ")"
-            rows.append((label, value, format_ratio(value, total)))
-    for key, value in stats.items():
-        if key.startswith("Frame"):
-            rows.append((key, value, format_ratio(value, total)))
-    return rows
-
-
-def emit_call_stats(stats, defines):
-    with Section("Call stats", summary="Inlined calls and frame stats"):
-        rows = calculate_call_stats(stats, defines)
-        emit_table(("", "Count:", "Ratio:"), rows)
-
-
-def emit_comparative_call_stats(base_stats, head_stats, defines):
-    with Section("Call stats", summary="Inlined calls and frame stats"):
-        base_rows = calculate_call_stats(base_stats, defines)
-        head_rows = calculate_call_stats(head_stats, defines)
-        rows = join_rows(base_rows, head_rows)
-        rows.sort(key=lambda x: -percentage_to_float(x[-1]))
-        emit_table(
-            ("", "Base Count:", "Base Ratio:", "Head Count:", "Head Ratio:"), rows
-        )
 
 
-def calculate_object_stats(stats):
-    total_materializations = stats.get("Object new values")
-    total_allocations = stats.get("Object allocations") + stats.get(
-        "Object allocations from freelist"
-    )
-    total_increfs = stats.get("Object interpreter increfs") + stats.get(
-        "Object increfs"
-    )
-    total_decrefs = stats.get("Object interpreter decrefs") + stats.get(
-        "Object decrefs"
+def call_stats_section() -> Section:
+    def calc_call_stats_table(stats: Stats) -> Rows:
+        call_stats = stats.get_call_stats()
+        total = sum(v for k, v in call_stats.items() if "Calls to" in k)
+        return [
+            (key, Count(value), Ratio(value, total))
+            for key, value in call_stats.items()
+        ]
+
+    return Section(
+        "Call stats",
+        "Inlined calls and frame stats",
+        [
+            Table(
+                ("", "Count:", "Ratio:"),
+                calc_call_stats_table,
+                JoinMode.CHANGE,
+            )
+        ],
     )
-    rows = []
-    for key, value in stats.items():
-        if key.startswith("Object"):
-            if "materialize" in key:
-                ratio = format_ratio(value, total_materializations)
-            elif "allocations" in key:
-                ratio = format_ratio(value, total_allocations)
-            elif "increfs" in key:
-                ratio = format_ratio(value, total_increfs)
-            elif "decrefs" in key:
-                ratio = format_ratio(value, total_decrefs)
-            else:
-                ratio = ""
-            label = key[6:].strip()
-            label = label[0].upper() + label[1:]
-            rows.append((label, value, ratio))
-    return rows
-
-
-def calculate_gc_stats(stats):
-    gc_stats = []
-    for key, value in stats.items():
-        if not key.startswith("GC"):
-            continue
-        n, _, rest = key[3:].partition("]")
-        name = rest.strip()
-        gen_n = int(n)
-        while len(gc_stats) <= gen_n:
-            gc_stats.append({})
-        gc_stats[gen_n][name] = value
-    return [
-        (i, gen["collections"], gen["objects collected"], gen["object visits"])
-        for (i, gen) in enumerate(gc_stats)
-    ]
-
-
-def emit_object_stats(stats):
-    with Section("Object stats", summary="allocations, frees and dict materializatons"):
-        rows = calculate_object_stats(stats)
-        emit_table(("", "Count:", "Ratio:"), rows)
-
-
-def emit_comparative_object_stats(base_stats, head_stats):
-    with Section("Object stats", summary="allocations, frees and dict materializatons"):
-        base_rows = calculate_object_stats(base_stats)
-        head_rows = calculate_object_stats(head_stats)
-        emit_table(
-            ("", "Base Count:", "Base Ratio:", "Head Count:", "Head Ratio:"),
-            join_rows(base_rows, head_rows),
-        )
 
 
-def emit_gc_stats(stats):
-    with Section("GC stats", summary="GC collections and effectiveness"):
-        rows = calculate_gc_stats(stats)
-        emit_table(
-            ("Generation:", "Collections:", "Objects collected:", "Object visits:"),
-            rows,
-        )
+def object_stats_section() -> Section:
+    def calc_object_stats_table(stats: Stats) -> Rows:
+        object_stats = stats.get_object_stats()
+        return [
+            (label, Count(value), Ratio(value, den))
+            for label, (value, den) in object_stats.items()
+        ]
 
+    return Section(
+        "Object stats",
+        "allocations, frees and dict materializatons",
+        [
+            Table(
+                ("", "Count:", "Ratio:"),
+                calc_object_stats_table,
+                JoinMode.CHANGE,
+            )
+        ],
+    )
 
-def emit_comparative_gc_stats(base_stats, head_stats):
-    with Section("GC stats", summary="GC collections and effectiveness"):
-        base_rows = calculate_gc_stats(base_stats)
-        head_rows = calculate_gc_stats(head_stats)
-        emit_table(
-            (
-                "Generation:",
-                "Base collections:",
-                "Head collections:",
-                "Base objects collected:",
-                "Head objects collected:",
-                "Base object visits:",
-                "Head object visits:",
-            ),
-            join_rows(base_rows, head_rows),
-        )
 
+def gc_stats_section() -> Section:
+    def calc_gc_stats(stats: Stats) -> Rows:
+        gc_stats = stats.get_gc_stats()
 
-def get_total(opcode_stats):
-    total = 0
-    for opcode_stat in opcode_stats.values():
-        if "execution_count" in opcode_stat:
-            total += opcode_stat["execution_count"]
-    return total
-
-
-def emit_pair_counts(opcode_stats, total):
-    pair_counts = []
-    for name_i, opcode_stat in opcode_stats.items():
-        for key, value in opcode_stat.items():
-            if key.startswith("pair_count"):
-                name_j, _, _ = key[11:].partition("]")
-                if value:
-                    pair_counts.append((value, (name_i, name_j)))
-    with Section("Pair counts", summary="Pair counts for top 100 pairs"):
-        pair_counts.sort(reverse=True)
-        cumulative = 0
-        rows = []
-        for count, pair in itertools.islice(pair_counts, 100):
-            name_i, name_j = pair
-            cumulative += count
-            rows.append(
-                (
-                    f"{name_i} {name_j}",
-                    count,
-                    format_ratio(count, total),
-                    format_ratio(cumulative, total),
-                )
+        return [
+            (
+                Count(i),
+                Count(gen["collections"]),
+                Count(gen["objects collected"]),
+                Count(gen["object visits"]),
             )
-        emit_table(("Pair", "Count:", "Self:", "Cumulative:"), rows)
-    with Section(
-        "Predecessor/Successor Pairs",
-        summary="Top 5 predecessors and successors of each opcode",
-    ):
-        predecessors = collections.defaultdict(collections.Counter)
-        successors = collections.defaultdict(collections.Counter)
-        total_predecessors = collections.Counter()
-        total_successors = collections.Counter()
-        for count, (first, second) in pair_counts:
-            if count:
-                predecessors[second][first] = count
-                successors[first][second] = count
-                total_predecessors[second] += count
-                total_successors[first] += count
-        for name in opcode_stats.keys():
-            total1 = total_predecessors[name]
-            total2 = total_successors[name]
-            if total1 == 0 and total2 == 0:
-                continue
-            pred_rows = succ_rows = ()
-            if total1:
-                pred_rows = [
-                    (pred, count, f"{count/total1:.1%}")
-                    for (pred, count) in predecessors[name].most_common(5)
-                ]
-            if total2:
-                succ_rows = [
-                    (succ, count, f"{count/total2:.1%}")
-                    for (succ, count) in successors[name].most_common(5)
-                ]
-            with Section(name, 3, f"Successors and predecessors for {name}"):
-                emit_table(("Predecessors", "Count:", "Percentage:"), pred_rows)
-                emit_table(("Successors", "Count:", "Percentage:"), succ_rows)
-
-
-def calculate_optimization_stats(stats):
-    attempts = stats["Optimization attempts"]
-    created = stats["Optimization traces created"]
-    executed = stats["Optimization traces executed"]
-    uops = stats["Optimization uops executed"]
-    trace_stack_overflow = stats["Optimization trace stack overflow"]
-    trace_stack_underflow = stats["Optimization trace stack underflow"]
-    trace_too_long = stats["Optimization trace too long"]
-    trace_too_short = stats["Optimiztion trace too short"]
-    inner_loop = stats["Optimization inner loop"]
-    recursive_call = stats["Optimization recursive call"]
-
-    return [
-        ("Optimization attempts", attempts, ""),
-        ("Traces created", created, format_ratio(created, attempts)),
-        ("Traces executed", executed, ""),
-        ("Uops executed", uops, int(uops / (executed or 1))),
-        ("Trace stack overflow", trace_stack_overflow, ""),
-        ("Trace stack underflow", trace_stack_underflow, ""),
-        ("Trace too long", trace_too_long, ""),
-        ("Trace too short", trace_too_short, ""),
-        ("Inner loop found", inner_loop, ""),
-        ("Recursive call", recursive_call, ""),
-    ]
-
-
-def calculate_uop_execution_counts(opcode_stats):
-    total = 0
-    counts = []
-    for name, opcode_stat in opcode_stats.items():
-        if "execution_count" in opcode_stat:
-            count = opcode_stat["execution_count"]
-            counts.append((count, name))
-            total += count
-    counts.sort(reverse=True)
-    cumulative = 0
-    rows = []
-    for count, name in counts:
-        cumulative += count
-        rows.append(
-            (name, count, format_ratio(count, total), format_ratio(cumulative, total))
-        )
-    return rows
+            for (i, gen) in enumerate(gc_stats)
+        ]
 
+    return Section(
+        "GC stats",
+        "GC collections and effectiveness",
+        [
+            Table(
+                ("Generation:", "Collections:", "Objects collected:", "Object visits:"),
+                calc_gc_stats,
+            )
+        ],
+    )
 
-def emit_optimization_stats(stats):
-    if "Optimization attempts" not in stats:
-        return
 
-    uop_stats = extract_opcode_stats(stats, "uops")
+def optimization_section() -> Section:
+    def calc_optimization_table(stats: Stats) -> Rows:
+        optimization_stats = stats.get_optimization_stats()
 
-    with Section(
-        "Optimization (Tier 2) stats", summary="statistics about the Tier 2 optimizer"
-    ):
-        with Section("Overall stats", level=3):
-            rows = calculate_optimization_stats(stats)
-            emit_table(("", "Count:", "Ratio:"), rows)
-
-        emit_histogram(
-            "Trace length histogram",
-            stats,
-            "Trace length",
-            stats["Optimization traces created"],
+        return [
+            (
+                label,
+                Count(value),
+                Ratio(value, den, percentage=label != "Uops executed"),
+            )
+            for label, (value, den) in optimization_stats.items()
+        ]
+
+    def calc_histogram_table(key: str, den: str) -> RowCalculator:
+        def calc(stats: Stats) -> Rows:
+            histogram = stats.get_histogram(key)
+            denominator = stats.get(den)
+
+            rows: Rows = []
+            last_non_zero = 0
+            for k, v in histogram:
+                if v != 0:
+                    last_non_zero = len(rows)
+                rows.append(
+                    (
+                        f"<= {k:,d}",
+                        Count(v),
+                        Ratio(v, denominator),
+                    )
+                )
+            # Don't include any zero entries at the end
+            rows = rows[: last_non_zero + 1]
+            return rows
+
+        return calc
+
+    def calc_unsupported_opcodes_table(stats: Stats) -> Rows:
+        unsupported_opcodes = stats.get_opcode_stats("unsupported_opcode")
+        return sorted(
+            [
+                (opcode, Count(count))
+                for opcode, count in unsupported_opcodes.get_opcode_counts().items()
+            ],
+            key=itemgetter(1),
+            reverse=True,
         )
-        emit_histogram(
-            "Optimized trace length histogram",
-            stats,
-            "Optimized trace length",
-            stats["Optimization traces created"],
+
+    def iter_optimization_tables(base_stats: Stats, head_stats: Stats | None = None):
+        if not base_stats.get_optimization_stats() or (
+            head_stats is not None and not head_stats.get_optimization_stats()
+        ):
+            return
+
+        yield Table(("", "Count:", "Ratio:"), calc_optimization_table, JoinMode.CHANGE)
+        for name, den in [
+            ("Trace length", "Optimization traces created"),
+            ("Optimized trace length", "Optimization traces created"),
+            ("Trace run length", "Optimization traces executed"),
+        ]:
+            yield Section(
+                f"{name} histogram",
+                "",
+                [
+                    Table(
+                        ("Range", "Count:", "Ratio:"),
+                        calc_histogram_table(name, den),
+                        JoinMode.CHANGE,
+                    )
+                ],
+            )
+        yield Section(
+            "Uop stats",
+            "",
+            [
+                Table(
+                    ("Name", "Count:", "Self:", "Cumulative:", "Miss ratio:"),
+                    calc_execution_count_table("uops"),
+                    JoinMode.CHANGE_ONE_COLUMN,
+                )
+            ],
         )
-        emit_histogram(
-            "Trace run length histogram",
-            stats,
-            "Trace run length",
-            stats["Optimization traces executed"],
+        yield Section(
+            "Unsupported opcodes",
+            "",
+            [
+                Table(
+                    ("Opcode", "Count:"),
+                    calc_unsupported_opcodes_table,
+                    JoinMode.CHANGE,
+                )
+            ],
         )
 
-        with Section("Uop stats", level=3):
-            rows = calculate_uop_execution_counts(uop_stats)
-            emit_table(("Uop", "Count:", "Self:", "Cumulative:"), rows)
-
-        with Section("Unsupported opcodes", level=3):
-            unsupported_opcodes = extract_opcode_stats(stats, "unsupported_opcode")
-            data = []
-            for opcode, entry in unsupported_opcodes.items():
-                data.append((entry["count"], opcode))
-            data.sort(reverse=True)
-            rows = [(x[1], x[0]) for x in data]
-            emit_table(("Opcode", "Count"), rows)
-
+    return Section(
+        "Optimization (Tier 2) stats",
+        "statistics about the Tier 2 optimizer",
+        iter_optimization_tables,
+    )
 
-def emit_comparative_optimization_stats(base_stats, head_stats):
-    print("## Comparative optimization stats not implemented\n\n")
 
+def meta_stats_section() -> Section:
+    def calc_rows(stats: Stats) -> Rows:
+        return [("Number of data files", Count(stats.get("__nfiles__")))]
 
-def output_single_stats(stats):
-    opcode_stats = extract_opcode_stats(stats, "opcode")
-    total = get_total(opcode_stats)
-    emit_execution_counts(opcode_stats, total)
-    emit_pair_counts(opcode_stats, total)
-    emit_specialization_stats(opcode_stats, stats["_defines"])
-    emit_specialization_overview(
-        opcode_stats, total, stats["_specialized_instructions"]
+    return Section(
+        "Meta stats",
+        "Meta statistics",
+        [Table(("", "Count:"), calc_rows, JoinMode.CHANGE)],
     )
-    emit_call_stats(stats, stats["_stats_defines"])
-    emit_object_stats(stats)
-    emit_gc_stats(stats)
-    emit_optimization_stats(stats)
-    with Section("Meta stats", summary="Meta statistics"):
-        emit_table(("", "Count:"), [("Number of data files", stats["__nfiles__"])])
-
 
-def output_comparative_stats(base_stats, head_stats):
-    base_opcode_stats = extract_opcode_stats(base_stats, "opcode")
-    base_total = get_total(base_opcode_stats)
 
-    head_opcode_stats = extract_opcode_stats(head_stats, "opcode")
-    head_total = get_total(head_opcode_stats)
-
-    emit_comparative_execution_counts(
-        base_opcode_stats, base_total, head_opcode_stats, head_total
-    )
-    emit_comparative_specialization_stats(
-        base_opcode_stats, head_opcode_stats, head_stats["_defines"]
-    )
-    emit_comparative_specialization_overview(
-        base_opcode_stats,
-        base_total,
-        head_opcode_stats,
-        head_total,
-        head_stats["_specialized_instructions"],
-    )
-    emit_comparative_call_stats(base_stats, head_stats, head_stats["_stats_defines"])
-    emit_comparative_object_stats(base_stats, head_stats)
-    emit_comparative_gc_stats(base_stats, head_stats)
-    emit_comparative_optimization_stats(base_stats, head_stats)
-
-
-def output_stats(inputs, json_output=None):
-    if len(inputs) == 1:
-        stats = gather_stats(inputs[0])
-        if json_output is not None:
-            json.dump(stats, json_output)
-        output_single_stats(stats)
-    elif len(inputs) == 2:
-        if json_output is not None:
-            raise ValueError("Can not output to JSON when there are multiple inputs")
-
-        base_stats = gather_stats(inputs[0])
-        head_stats = gather_stats(inputs[1])
-        output_comparative_stats(base_stats, head_stats)
-
-    print("---")
-    print("Stats gathered on:", date.today())
+LAYOUT = [
+    execution_count_section(),
+    pair_count_section(),
+    pre_succ_pairs_section(),
+    specialization_section(),
+    specialization_effectiveness_section(),
+    call_stats_section(),
+    object_stats_section(),
+    gc_stats_section(),
+    optimization_section(),
+    meta_stats_section(),
+]
+
+
+def output_markdown(
+    out: TextIO,
+    obj: Section | Table | list,
+    base_stats: Stats,
+    head_stats: Stats | None = None,
+    level: int = 2,
+) -> None:
+    def to_markdown(x):
+        if hasattr(x, "markdown"):
+            return x.markdown()
+        elif isinstance(x, str):
+            return x
+        elif x is None:
+            return ""
+        else:
+            raise TypeError(f"Can't convert {x} to markdown")
+
+    match obj:
+        case Section():
+            if obj.title:
+                print("#" * level, obj.title, file=out)
+                print(file=out)
+                print("<details>", file=out)
+                print("<summary>", obj.summary, "</summary>", file=out)
+                print(file=out)
+            if head_stats is not None and obj.comparative is False:
+                print("Not included in comparative output.\n")
+            else:
+                for part in obj.part_iter(base_stats, head_stats):
+                    output_markdown(out, part, base_stats, head_stats, level=level + 1)
+            print(file=out)
+            if obj.title:
+                print("</details>", file=out)
+                print(file=out)
+
+        case Table():
+            header, rows = obj.get_table(base_stats, head_stats)
+            if len(rows) == 0:
+                return
+
+            width = len(header)
+            header_line = "|"
+            under_line = "|"
+            for item in header:
+                under = "---"
+                if item.endswith(":"):
+                    item = item[:-1]
+                    under += ":"
+                header_line += item + " | "
+                under_line += under + "|"
+            print(header_line, file=out)
+            print(under_line, file=out)
+            for row in rows:
+                if len(row) != width:
+                    raise ValueError(
+                        "Wrong number of elements in row '" + str(row) + "'"
+                    )
+                print("|", " | ".join(to_markdown(i) for i in row), "|", file=out)
+            print(file=out)
+
+        case list():
+            for part in obj:
+                output_markdown(out, part, base_stats, head_stats, level=level)
+
+            print("---", file=out)
+            print("Stats gathered on:", date.today(), file=out)
+
+
+def output_stats(inputs: list[Path], json_output=TextIO | None):
+    match len(inputs):
+        case 1:
+            data = load_raw_data(Path(inputs[0]))
+            if json_output is not None:
+                save_raw_data(data, json_output)  # type: ignore
+            stats = Stats(data)
+            output_markdown(sys.stdout, LAYOUT, stats)
+        case 2:
+            if json_output is not None:
+                raise ValueError(
+                    "Can not output to JSON when there are multiple inputs"
+                )
+            base_data = load_raw_data(Path(inputs[0]))
+            head_data = load_raw_data(Path(inputs[1]))
+            base_stats = Stats(base_data)
+            head_stats = Stats(head_data)
+            output_markdown(sys.stdout, LAYOUT, base_stats, head_stats)
 
 
 def main():