#!/usr/bin/env python3 """Generate publication-ready Raptor vs PIMCOMP comparison charts. Usage: .venv/bin/python validation/tools/pim/pimcomp/compare/generate_plots.py Dependencies: matplotlib (no pandas or seaborn required). By default, reads results_comparison.csv beside this script and writes plots/ beside it, with outputs grouped in png/, svg/, and pdf/ under the output directory. YOLO models without PIMCOMP measurements get separate Raptor-only charts. Comparisons use vertical bars for ratios, diverging advantages, and absolute values, with compiler colors and patterns for the Raptor pipeline stages. A compact dot-plot summary combines latency, throughput, and both energy cases. Throughput-mode latency is the output interval, a bottleneck-stage proxy, not end-to-end inference latency. Separate throughput plots use inferences/s. Energy is reported per inference; power is averaged over the simulation. Numerical results whose compiler did not pass functional validation have a red status following their value label; missing PIMCOMP measurements appear as 'N/A'. The fixed compiler palette is shared by all metrics. Relative plots use PIMCOMP / Raptor for latency, power and energy, and Raptor / PIMCOMP for throughput, so >1 favors Raptor. Diverging plots put the winner's factor above (Raptor) or below (PIMCOMP) parity, with 1x at the zero line. """ from __future__ import annotations import argparse import csv import math import sys from dataclasses import dataclass from itertools import groupby from pathlib import Path import matplotlib matplotlib.use("Agg") import matplotlib.pyplot as plt from matplotlib.patches import Patch, Rectangle from matplotlib.lines import Line2D from matplotlib.text import Text from matplotlib.ticker import FuncFormatter, MultipleLocator from matplotlib.transforms import Bbox sys.path.insert(0, str(Path(__file__).resolve().parents[2])) from plot_style import (ARCH_NAMES, ARCH_STYLES, BAND_COLOR, COMPACT_MODE_NAMES, COMPILER_NAMES, COMPILER_STYLES, DAGGER_COLOR, ERROR_COLOR, LINE_COLOR, MATRIX_TEXT_COLOR, MODE_NAMES, MODEL_NAMES, NETWORK_STYLES, PALETTE, TEXT_COLOR, add_legend, configure_style, fit_value_labels, fit_bar_widths, fit_x_tick_labels, save_figure, value_label) STAGE_PATTERNS = {1: "", 2: "..", 4: "//", 8: "\\\\"} ARCHITECTURES = ("arch-a", "arch-b") MODELS = ("vgg8", "resnet18", "resnet34", "googlenet", "yolo11n") CONFIGURATIONS = (("latency", 1), ("throughput", 2), ("throughput", 4), ("throughput", 8)) VALIDATION_STYLES = { "PASS": PALETTE["sage"], "FAIL": PALETTE["red"], "SKIPPED": PALETTE["pale_sand"], "NA": PALETTE["pale_blue"], } @dataclass(frozen=True) class Result: arch: str model: str mode: str pipeline: int pimcomp_schedule: str raptor_validation: str pimcomp_validation: str latency_speedup: float power: float energy: float raptor_latency: float pimcomp_latency: float raptor_throughput: float pimcomp_throughput: float raptor_power: float pimcomp_power: float raptor_energy: float pimcomp_energy: float @property def comparison_passed(self) -> bool: return self.raptor_validation == self.pimcomp_validation == "PASS" @property def throughput_speedup(self) -> float: return quotient(self.raptor_throughput, self.pimcomp_throughput) @property def speedup(self) -> float: return self.latency_speedup if self.mode == "latency" else self.throughput_speedup def positive_float(text: str | None) -> float: try: value = float(text) except (TypeError, ValueError): return math.nan return value if math.isfinite(value) and value > 0 else math.nan def quotient(numerator: float, denominator: float) -> float: if not (math.isfinite(numerator) and math.isfinite(denominator)): return math.nan return numerator / denominator def validation_status(text: str) -> str: status = text.strip().upper() if status not in VALIDATION_STYLES: raise ValueError(f"Unknown functional-validation status: {text!r}") return status def load_results(path: Path) -> list[Result]: with path.open(newline="", encoding="utf-8-sig") as handle: source = list(csv.DictReader(handle)) rows = [] expected = {(arch, model, mode, stage) for arch in ARCHITECTURES for model in MODELS for mode, stage in CONFIGURATIONS} seen = {} for line, row in enumerate(source, 2): try: arch, model, mode = (row[field].strip().lower() for field in ("arch", "model", "mode")) stage = int(row["raptor_pipeline"]) except (KeyError, AttributeError, TypeError, ValueError) as exc: raise ValueError(f"Invalid configuration key at CSV row {line}: {exc}") from exc key = arch, model, mode, stage if key not in expected: raise ValueError(f"Unexpected configuration at CSV row {line}: {key}") if key in seen: raise ValueError(f"Duplicate configuration {key} at CSV rows {seen[key]} and {line}") seen[key] = line schedule = row["pimcomp_pipeline"].strip().lower() if schedule != ("element" if mode == "latency" else "batch"): raise ValueError(f"{key}: incompatible PIMCOMP schedule {schedule!r}") rows.append(Result( arch=arch, model=model, mode=mode, pipeline=stage, pimcomp_schedule=schedule, raptor_validation=validation_status(row["raptor_functional_validation"]), pimcomp_validation=validation_status(row["pimcomp_functional_validation"]), latency_speedup=quotient(positive_float(row["pimcomp_latency_ms"]), positive_float(row["raptor_latency_ms"])), power=quotient(positive_float(row["pimcomp_power_mw"]), positive_float(row["raptor_power_mw"])), energy=quotient(positive_float(row["pimcomp_energy_pj"]), positive_float(row["raptor_energy_pj"])), raptor_latency=positive_float(row["raptor_latency_ms"]), pimcomp_latency=positive_float(row["pimcomp_latency_ms"]), raptor_throughput=positive_float(row["raptor_throughput_samples_s"]), pimcomp_throughput=positive_float(row["pimcomp_throughput_samples_s"]), raptor_power=positive_float(row["raptor_power_mw"]), pimcomp_power=positive_float(row["pimcomp_power_mw"]), raptor_energy=positive_float(row["raptor_energy_pj"]), pimcomp_energy=positive_float(row["pimcomp_energy_pj"]), )) missing = expected - seen.keys() if missing: raise ValueError(f"Missing {len(missing)} expected configurations. Examples: {sorted(missing)[:5]}") rows.sort(key=lambda r: (ARCHITECTURES.index(r.arch), MODELS.index(r.model), r.pipeline)) absolute_columns(rows) return rows def advantage_compiler(ratio: float) -> str | None: if ratio > 1: return "raptor" if ratio < 1: return "pimcomp" return None def figure_legend(fig, results: list[Result], field: str, absolute=False, raptor_only=False): compilers = ("raptor",) if raptor_only else ("raptor", "pimcomp") if absolute: measurements = [(row, compiler, getattr(row, f"{compiler}_{field}")) for row, compiler in absolute_columns(results, raptor_only)] else: measurements = [(row, None, getattr(row, field)) for row in results] available = [(row, compiler, value) for row, compiler, value in measurements if math.isfinite(value)] stages = {row.pipeline for row, compiler, _ in available if compiler != "pimcomp"} handles = [] visible_compilers = {compiler if absolute else advantage_compiler(value) for _, compiler, value in available} - {None} if len(visible_compilers) > 1: for compiler in compilers: fill, border = COMPILER_STYLES[compiler] handles.append(Patch(facecolor=fill, edgecolor=border, label=COMPILER_NAMES[compiler] + ("" if absolute else " advantage"))) handles += [Patch(facecolor="white", edgecolor=TEXT_COLOR, hatch=STAGE_PATTERNS[depth], label=("1 stage (latency)" if absolute else "Single-inference") if depth == 1 else f"{depth} stages") for depth in sorted(stages) if stages != {1}] styles = [] if len(available) < len(measurements): styles.append(Patch(facecolor="white", edgecolor=LINE_COLOR, label="N/A: missing measurement")) handles += styles if handles: add_legend(fig, handles) def absolute_columns(results: list[Result], raptor_only: bool = False) -> list[tuple[Result, str]]: columns = [] for (arch, model), group in groupby(results, key=lambda row: (row.arch, row.model)): rows = sorted(group, key=lambda row: row.pipeline) for row in rows: columns.append((row, "raptor")) if row.mode == "latency" and not raptor_only: columns.append((row, "pimcomp")) throughput = [row for row in rows if row.mode == "throughput"] if throughput and not raptor_only: signatures = { (row.pimcomp_schedule, row.pimcomp_validation, *(None if math.isnan(value) else value for value in ( row.pimcomp_latency, row.pimcomp_throughput, row.pimcomp_power, row.pimcomp_energy))) for row in throughput } if len(signatures) != 1: raise ValueError(f"{arch}/{model}: inconsistent shared PIMCOMP batch results across pipelines") columns.append((throughput[0], "pimcomp")) return columns def label_groups(ax, results: list[Result], positions: list[float] | None = None) -> None: if positions is None: positions = list(range(len(results))) centers, labels = [], [] start = 0 for index, (model, rows) in enumerate(groupby(results, key=lambda row: row.model)): end = start + len(list(rows)) centers.append((positions[start] + positions[end - 1]) / 2) labels.append(MODEL_NAMES[model]) left = ax.get_xlim()[0] if start == 0 else (positions[start - 1] + positions[start]) / 2 right = ax.get_xlim()[1] if end == len(results) else (positions[end - 1] + positions[end]) / 2 if index % 2 == 0: ax.axvspan(left, right, facecolor=BAND_COLOR, zorder=0) if index > 0: ax.axvline(left, color=LINE_COLOR, linewidth=0.7) start = end ax.set_xticks(centers, labels, fontsize=9.5) def grouped_positions(results: list[Result]) -> list[float]: positions = [] for index, row in enumerate(results): if index == 0: positions.append(0.0) continue previous = results[index - 1] gap = 0.8 if row.model != previous.model else 0.4 if row.mode != previous.mode else 0.0 positions.append(positions[-1] + 1.0 + gap) return positions def diverging_ratio(ratio: float) -> float: return ratio - 1 if ratio >= 1 else 1 - 1 / ratio def draw_validation(results: list[Result], output_dir: Path, compact: bool = False) -> None: cells = {} for row, compiler in absolute_columns(results): stage = row.pipeline if compiler == "raptor" else row.pimcomp_schedule key = (row.arch, row.model, row.mode, None if compact else stage, compiler) cells.setdefault(key, set()).add(getattr(row, f"{compiler}_validation")) configurations = list(dict.fromkeys(key[2:] for key in cells)) positions = [] for index, (mode, _, _) in enumerate(configurations): gap = 0.4 if index and mode != configurations[index - 1][0] else 0 positions.append(positions[-1] + 1 + gap if index else 0) configuration_x = dict(zip(configurations, positions)) headers = [f"{COMPILER_NAMES['raptor']}\n{stage} stages" if mode == "throughput" and compiler == "raptor" and not compact else "PIMCOMP\nbatch" if mode == "throughput" and compiler == "pimcomp" and not compact else COMPILER_NAMES[compiler] for mode, stage, compiler in configurations] fig, axes = plt.subplots(1, 2, figsize=(8.5 if compact else 12, 3.3), layout="constrained") for ax, arch in zip(axes, ARCHITECTURES): ax.set_xlim(positions[0] - 0.55, positions[-1] + 0.55) ax.set_ylim(len(MODELS) - 0.5, -0.5) centers, modes = [], [] for mode, group in groupby(zip(configurations, positions), key=lambda item: item[0][0]): group_positions = [x for _, x in group] centers.append((group_positions[0] + group_positions[-1]) / 2) modes.append(COMPACT_MODE_NAMES[mode] if compact else "Latency" if mode == "latency" else MODE_NAMES[mode]) ax.set_xticks(centers, modes, fontsize=10) ax.set_xticks(positions, headers, minor=True) ax.tick_params(axis="x", which="minor", top=True, labeltop=True, bottom=False, labelbottom=False, length=0, pad=6, labelsize=9) ax.set_yticks(range(len(MODELS)), [MODEL_NAMES[model] for model in MODELS], fontsize=9) ax.tick_params(axis="both", length=0, pad=6) ax.tick_params(axis="y", labelleft=arch == ARCHITECTURES[0]) architecture = ARCH_NAMES[arch] ax.set_xlabel(architecture, fontsize=11, weight="bold", labelpad=8) for (cell_arch, model, mode, stage, compiler), statuses in cells.items(): if cell_arch != arch: continue x, y = configuration_x[(mode, stage, compiler)], MODELS.index(model) status = next(status for status in ("FAIL", "SKIPPED", "NA", "PASS") if status in statuses) color, border = VALIDATION_STYLES[status] ax.add_patch(Rectangle((x - 0.5, y - 0.42), 1, 0.84, facecolor=color, edgecolor=border, linewidth=0.7)) ax.text(x, y, "N/A" if status == "NA" else status, color=MATRIX_TEXT_COLOR, ha="center", va="center", fontsize=12, weight="normal") fit_x_tick_labels(fig) save_figure(fig, "validation_status_compact" if compact else "validation_status", output_dir) def select_compact_speedups(results: list[Result]) -> list[Result]: selected = [] for _, group in groupby(results, key=lambda row: (row.arch, row.model, row.mode)): rows = list(group) selected.append(max(rows, key=lambda row: row.speedup if math.isfinite(row.speedup) else -math.inf)) return selected def draw_compact_summary(results: list[Result], output_dir: Path, *, marker_borders: bool = True) -> None: selected = select_compact_speedups([row for row in results if row.model != "yolo11n"]) lookup = {(row.model, row.arch, row.mode): row for row in selected} keys = [(model, arch) for model in MODELS if model != "yolo11n" for arch in ARCHITECTURES] panels = ((f"{COMPACT_MODE_NAMES['latency']}\nspeedup", "latency", "latency_speedup"), (f"{COMPACT_MODE_NAMES['throughput']}\nspeedup", "throughput", "throughput_speedup"), (f"{COMPACT_MODE_NAMES['latency']}\nenergy reduction", "latency", "energy"), (f"{COMPACT_MODE_NAMES['throughput']}\nenergy reduction", "throughput", "energy")) measurements = [] for _, mode, field in panels: points = [] for model, arch in keys: row = lookup[(model, arch, mode)] ratio = getattr(row, field) if not math.isfinite(ratio) or ratio <= 0: raise ValueError(f"Compact summary requires a positive {field} ratio for {arch}/{model}") pimcomp_invalid = row.pimcomp_schedule == "batch" or row.pimcomp_validation == "FAIL" status = ("P. INVALID" if pimcomp_invalid else f"{COMPILER_NAMES['raptor'][0]}. INVALID" if row.raptor_validation == "FAIL" else "NOT VALIDATED" if not row.comparison_passed else "") points.append((row, ratio, status)) measurements.append(points) ratios = [ratio for points in measurements for _, ratio, _ in points] padding = max(max(ratios) - min(ratios), 1) * 0.04 limits = (0.25 if min(ratios) < 0.5 else 0.5, max(1, *ratios) + padding) fig, axes = plt.subplots(4, 1, sharex=True, figsize=(6.8, 7.6), layout="constrained") markers = dict(zip((model for model in MODELS if model != "yolo11n"), ("o", "s", "^", "D"))) lanes = {arch: index * 2 for index, arch in enumerate(ARCHITECTURES)} daggers = {} for ax, (title, _, _), points in zip(axes, panels, measurements): ax.set_xlim(*limits) ax.set_ylim(4, -2) ax.set_ylabel(title, rotation=90, ha="center", va="bottom", multialignment="center", fontsize=8.5, labelpad=10) ax.set_yticks(list(lanes.values()), [ARCH_NAMES[arch] for arch in lanes], fontsize=8) ax.set_xticks([limits[0], *range(1, math.floor(limits[1]) + 1)]) ax.xaxis.set_major_formatter(FuncFormatter(lambda value, _: f"{value:g}")) ax.tick_params(axis="both", length=0, pad=4, labelsize=8.5) ax.tick_params(axis="x", bottom=False, labelbottom=False, top=ax is axes[0], labeltop=ax is axes[0]) ax.grid(axis="x", linewidth=0.6) ax.axvline(1, color=TEXT_COLOR, linewidth=0.9, linestyle="--", zorder=1) for lane in lanes.values(): ax.axhline(lane, color=LINE_COLOR, linewidth=0.6) ax.spines["left"].set_visible(False) ax.spines["bottom"].set_visible(False) ax.spines["top"].set_visible(ax is axes[0]) for row, ratio, status in points: fill, border = NETWORK_STYLES[row.model] lane = lanes[row.arch] ax.plot(ratio, lane, marker=markers[row.model], markersize=6.5, linestyle="none", markerfacecolor=fill if marker_borders else border, markeredgecolor=border, markeredgewidth=1.1 if marker_borders else 0) suffix = f" [{status}]" if status and status != "P. INVALID" else "" label = f"{ratio:.2f}×{suffix}" annotation = ax.annotate(label, (ratio, lane), xytext=(0, -5), textcoords="offset points", rotation=90, rotation_mode="anchor", ha="right", va="center", fontsize=7.5, color=TEXT_COLOR, math_fontfamily="cm") annotation.set_in_layout(False) dagger = ax.annotate(r"$\boldsymbol{\dagger}$", (0.5, 1), xycoords=annotation, xytext=(0, 1), textcoords="offset points", rotation=90, rotation_mode="anchor", ha="left", va="center", fontsize=9, color=DAGGER_COLOR, math_fontfamily="cm", alpha=1 if status == "P. INVALID" else 0) dagger.set_in_layout(False) daggers[annotation] = dagger handles = [Line2D([], [], marker=marker, color=NETWORK_STYLES[model][1], markerfacecolor=NETWORK_STYLES[model][0 if marker_borders else 1], markersize=6.5, markeredgewidth=1 if marker_borders else 0, linestyle="none", label=MODEL_NAMES[model]) for model, marker in markers.items()] handles.append(Text(0, 0, r"$\boldsymbol{\dagger}$", color=DAGGER_COLOR, fontsize=9, math_fontfamily="cm", label="PIMCOMP invalid")) add_legend(fig, handles) axes[0].xaxis.set_label_position("top") axes[0].set_xlabel("Ratio (x)", fontsize=9, labelpad=7, va="bottom") axes[0].xaxis.set_label_coords(0.5, 1.18) axes[0].text(0, 1.18, "← PIMCOMP better", transform=axes[0].transAxes, ha="left", va="bottom", fontsize=8) axes[0].text(1, 1.18, f"{COMPILER_NAMES['raptor']} better →", transform=axes[0].transAxes, ha="right", va="bottom", fontsize=8) fig.canvas.draw() renderer = fig.canvas.get_renderer() def label_bounds(text): return Bbox.union([text.get_window_extent(renderer), daggers[text].get_window_extent(renderer)]) def position_label(text, dx, dy): text.set_ha("left" if dy > 0 else "right") extra = (daggers[text].get_window_extent(renderer).height + renderer.points_to_pixels(1) if dy < 0 else 0) text.set_position((dx, dy - extra * 72 / fig.dpi)) labels_by_axis = [[text for text in ax.texts if getattr(text, "xycoords", None) == "data"] for ax in axes] label_height = max(label_bounds(text).height for labels in labels_by_axis for text in labels) lane_gap = min(abs(ax.transData.transform((0, 0))[1] - ax.transData.transform((0, 2))[1]) for ax in axes) height_scale = (label_height + renderer.points_to_pixels(5)) / lane_gap if height_scale > 1: fig.set_size_inches(fig.get_figwidth(), fig.get_figheight() * height_scale) fig.canvas.draw() renderer = fig.canvas.get_renderer() for ax, labels in zip(axes, labels_by_axis): placed = [] for line in ax.lines: placed.append(line.get_window_extent(renderer).padded(1)) bounds = ax.get_window_extent(renderer) choices = [] for text in labels: preferred_dy = 5 if text.xy[1] == lanes[ARCHITECTURES[0]] else -5 candidates = [] for dx in (0, -3, 3, -6, 6, -8, 8): for dy in (-5, 5): position_label(text, dx, dy) box = label_bounds(text) overlap = any(box.overlaps(other) for other in placed) outside = not (bounds.x0 <= box.x0 <= box.x1 <= bounds.x1 and bounds.y0 <= box.y0 <= box.y1 <= bounds.y1) if not outside and not overlap: candidates.append(((dy != preferred_dy, abs(dx) + abs(dy)), dx, dy, box)) choices.append((text, candidates)) layouts = [((0, 0), [])] for text, candidates in sorted(choices, key=lambda choice: len(choice[1])): layouts = sorted( (((cost[0] + distance[0], cost[1] + distance[1]), assignments + [(text, dx, dy, box)]) for cost, assignments in layouts for distance, dx, dy, box in candidates if not any(box.overlaps(other[3]) for other in assignments)), key=lambda layout: layout[0])[:32] if not layouts: raise ValueError("Dot labels do not fit without overlap; increase lane spacing or figure width") for text, dx, dy, _ in layouts[0][1]: position_label(text, dx, dy) save_figure(fig, "performance_energy_compact", output_dir) def draw_metric(results: list[Result], field: str, ylabel: str, basename: str, output_dir: Path, diverging: bool = False, compact: bool = False) -> None: data = [diverging_ratio(getattr(row, field)) if diverging else getattr(row, field) for row in results] available = [v for v in data if math.isfinite(v)] upper = max(0, max(available, default=1.0)) lower = (min(0, min(available, default=0)) if diverging else 0.25 if min(available, default=1) < 0.5 else 0.5) padding = max(upper - lower, 1.0) * 0.3 y_max = math.ceil((upper + padding) * 2) / 2 y_min = math.floor((lower - padding) * 2) / 2 if diverging else lower y_interval = 1.0 if y_max - y_min > 4.5 else 0.5 fig, axes = plt.subplots(1, 2, sharey=True, figsize=(8.5 if compact else 12, 4.4), layout="constrained") if compact: metric_suffix = "" if field == "speedup" else f" {field}" panel_labels = {mode: label + metric_suffix for mode, label in COMPACT_MODE_NAMES.items()} handles = [Patch(facecolor=fill, edgecolor=border, hatch=pattern, label=ARCH_NAMES[arch]) for arch, (fill, border, pattern) in ARCH_STYLES.items()] panels = ("latency", "throughput") if compact else ARCHITECTURES for panel_index, (ax, panel) in enumerate(zip(axes, panels)): selected = [row for row in results if getattr(row, "mode" if compact else "arch") == panel] if compact: selected.sort(key=lambda row: (MODELS.index(row.model), ARCHITECTURES.index(row.arch))) positions = [index + index // 2 for index in range(len(selected))] if compact else grouped_positions(selected) ax.set_xlabel(panel_labels[panel] if compact else ARCH_NAMES[panel], fontsize=11, weight="bold", labelpad=8) ax.set_axisbelow(True) ax.set_ylim(y_min, y_max) ax.set_xlim(-0.9, positions[-1] + 0.9 if positions else 0.9) label_groups(ax, selected, positions) ax.tick_params(axis="y", labelleft=panel_index == 0, pad=5, length=0) ax.tick_params(axis="x", labelsize=9.5, length=0, pad=6) ax.yaxis.set_major_locator(MultipleLocator(y_interval)) if diverging: ax.yaxis.set_major_formatter(FuncFormatter(lambda value, _: f"{abs(value) + 1:g}")) else: ax.set_yticks([y_min], minor=True) ax.yaxis.set_minor_formatter(FuncFormatter(lambda value, _: f"{value:g}")) ax.tick_params(axis="y", which="minor", labelleft=panel_index == 0, pad=5, length=0) ax.grid(axis="y") ax.axhline(0 if diverging else 1, color=TEXT_COLOR, linewidth=1.0, linestyle="--", zorder=3) if panel_index == 0: ax.set_ylabel(ylabel, fontsize=10.5, labelpad=6) for position, row in zip(positions, selected): ratio = getattr(row, field) if not math.isfinite(ratio): ax.text(position, (0 if diverging else y_min) + (y_max - y_min) * 0.02, "N/A", ha="center", va="bottom", fontsize=9.5, color=TEXT_COLOR, fontstyle="italic") continue compiler = advantage_compiler(ratio) statuses = [status for status in (row.raptor_validation, row.pimcomp_validation) if status != "PASS"] status = "FAIL" if "FAIL" in statuses else next(iter(statuses), "PASS") pattern = STAGE_PATTERNS[row.pipeline] color, border = COMPILER_STYLES[compiler] if compiler else ("white", TEXT_COLOR) if compact: color, border, pattern = ARCH_STYLES[row.arch] measurement = diverging_ratio(ratio) if diverging else ratio ax.bar(position, measurement, width=1.0, color=color, hatch=pattern, edgecolor=border, linewidth=0.7, zorder=2) direction = -1 if measurement < 0 else 1 factor = max(ratio, 1 / ratio) if diverging else ratio value_label(ax, position, measurement + direction * (y_max - y_min) * 0.012, f"{factor:.2f}×", status, direction, fontsize=9.1, invalid_label="P. INVALID") fit_bar_widths(fig) if compact: add_legend(fig, handles) else: figure_legend(fig, results, field) fit_x_tick_labels(fig) fit_value_labels(fig, lower_limit=None if diverging else y_min) save_figure(fig, basename, output_dir) def draw_absolute(results: list[Result], field: str, ylabel: str, basename: str, output_dir: Path, scale: float = 1.0, raptor_only: bool = False) -> None: columns = absolute_columns(results, raptor_only) available = [getattr(row, f"{compiler}_{field}") * scale for row, compiler in columns if math.isfinite(getattr(row, f"{compiler}_{field}"))] y_max = max(available, default=1.0) * 1.35 fig, axes = plt.subplots(1, 2, sharey=True, figsize=(12, 4.4), layout="constrained") for ax, arch in zip(axes, ARCHITECTURES): selected = [(row, compiler) for row, compiler in columns if row.arch == arch] positions = grouped_positions([row for row, _ in selected]) ax.set_xlabel(ARCH_NAMES[arch], fontsize=11, weight="bold", labelpad=8) ax.set_axisbelow(True) ax.set_ylim(0, y_max) if positions: ax.set_xlim(positions[0] - 0.9, positions[-1] + 0.9) else: ax.set_xlim(-0.9, 0.9) label_groups(ax, [row for row, _ in selected], positions) if {row.mode for row, _ in selected} == {"latency", "throughput"}: centers, labels = [], [] for (_, mode), group in groupby(enumerate(selected), key=lambda item: (item[1][0].model, item[1][0].mode)): group_positions = [positions[position] for position, _ in group] centers.append((group_positions[0] + group_positions[-1]) / 2) labels.append("Latency" if mode == "latency" else MODE_NAMES[mode]) ax.set_xticks(centers, labels, minor=True) ax.tick_params(axis="x", which="minor", length=0, pad=4, labelsize=8) if arch == ARCHITECTURES[0]: ax.set_ylabel(ylabel, fontsize=10.5) if field == "throughput": ax.ticklabel_format(axis="y", style="plain", useOffset=False) ax.grid(axis="y") ax.tick_params(axis="y", labelleft=arch == ARCHITECTURES[0], length=0, pad=6) ax.tick_params(axis="x", which="major", length=0, pad=22 if {row.mode for row, _ in selected} == {"latency", "throughput"} else 6) for x, (row, compiler) in zip(positions, selected): measurement = getattr(row, f"{compiler}_{field}") * scale if not math.isfinite(measurement): ax.text(x, y_max * 0.02, "N/A", rotation=90, ha="center", va="bottom", fontsize=8, color=TEXT_COLOR) continue status = getattr(row, f"{compiler}_validation") pattern = STAGE_PATTERNS[row.pipeline] if compiler == "raptor" else "" fill, border = COMPILER_STYLES[compiler] ax.bar(x, measurement, width=1.0, color=fill, hatch=pattern, edgecolor=border, linewidth=0.7, zorder=2) label = (f"{measurement:,.2f}".rstrip("0").rstrip(".") if field == "throughput" else f"{measurement:.3g}") value_label(ax, x, measurement + y_max * 0.012, label, status, invalid_label="P. INVALID") fit_bar_widths(fig) figure_legend(fig, results, field, absolute=True, raptor_only=raptor_only) fit_x_tick_labels(fig) fit_value_labels(fig) save_figure(fig, basename, output_dir) def draw_pipeline_effect(results: list[Result], output_dir: Path, *, marker_borders: bool = True) -> None: rows = [row for row in results if row.mode == "throughput" or row.pipeline == 1] stages = sorted({row.pipeline for row in rows}) panels = [model for model in MODELS if any(row.model == model for row in rows)] lookup = {(row.arch, row.model, row.pipeline): row for row in rows} rates = {key: quotient(1000, row.raptor_latency) if row.mode == "latency" else row.raptor_throughput for key, row in lookup.items()} fig, axes = plt.subplots(len(ARCHITECTURES), len(panels), sharex=True, sharey="col", figsize=(2.6 * len(panels), 4.3), squeeze=False, layout="constrained") fig.supylabel("Throughput (inferences/s)", fontsize=10.5) for column, model in enumerate(panels): fill, border = NETWORK_STYLES[model] for index, arch in enumerate(ARCHITECTURES): ax = axes[index, column] keys = [(arch, model, stage) for stage in stages] values = [rates.get(key, math.nan) for key in keys] ax.plot(stages, values, color=border, linewidth=1.8) for stage, value in zip(stages, values): if math.isfinite(value): ax.plot(stage, value, color=border, markerfacecolor=fill if marker_borders else border, marker="s" if stage == 1 else "o", markeredgewidth=1 if marker_borders else 0, markersize=7, linestyle="none") ax.set_xticks(stages) if index == 0: ax.set_title(MODEL_NAMES[model], fontsize=10.5, weight="normal", pad=8) ax.grid(axis="y") ax.tick_params(axis="both", length=0, pad=6, labelbottom=index == len(ARCHITECTURES) - 1, labelleft=True) ax.ticklabel_format(axis="y", style="plain", useOffset=False) if column == len(panels) - 1: ax.yaxis.set_label_position("right") ax.set_ylabel(ARCH_NAMES[arch], fontsize=10.5, weight="bold", rotation=270, labelpad=24) for ax in axes.flat: ax.margins(y=0.08) handles = [Line2D([], [], color=TEXT_COLOR, markerfacecolor="white" if marker_borders else TEXT_COLOR, markeredgewidth=1 if marker_borders else 0, marker=marker, markersize=7, linestyle="none", label=label) for marker, label in (("s", "Derived as 1 / latency"), ("o", "Throughput measurement"))] add_legend(fig, handles) fig.supxlabel(f"{COMPILER_NAMES['raptor']} pipeline stages", fontsize=11) fit_x_tick_labels(fig) save_figure(fig, "raptor_pipeline_throughput", output_dir) def main() -> None: parser = argparse.ArgumentParser(description=__doc__) script_dir = Path(__file__).resolve().parent parser.add_argument("--input", type=Path, default=script_dir / "results_comparison.csv") parser.add_argument("--output-dir", type=Path, default=script_dir / "plots/default") parser.add_argument("--colors", action=argparse.BooleanOptionalAction, default=True, help="Use the pastel palette (default); --no-colors exports grayscale charts") parser.add_argument("--anonymize", action=argparse.BooleanOptionalAction, default=False, help="Replace the displayed Raptor name with ANON") parser.add_argument("--marker-borders", action=argparse.BooleanOptionalAction, default=True, help="Outline markers (default); --no-marker-borders uses solid markers in dot and line charts") args = parser.parse_args() configure_style(colors=args.colors, anonymize=args.anonymize) results = load_results(args.input) valid = sum(row.comparison_passed for row in results) print(f"Results: {len(results)} configurations, {valid} valid comparisons, " f"{len(results)-valid} comparisons without two PASS statuses") draw_validation(results, args.output_dir) draw_validation(results, args.output_dir, compact=True) draw_pipeline_effect(results, args.output_dir, marker_borders=args.marker_borders) yolo_only_models = { model for model in MODELS if model.startswith("yolo") and not any(math.isfinite(getattr(row, f"pimcomp_{field}")) for row in results if row.model == model for field in ("latency", "power", "energy")) } comparisons = [row for row in results if row.model not in yolo_only_models] yolo_only = [row for row in results if row.model in yolo_only_models] compact_speedups = select_compact_speedups(comparisons) draw_compact_summary(comparisons, args.output_dir, marker_borders=args.marker_borders) for metric, label in (("speedup", "Speedup (x)"), ("energy", "Energy reduction (x)"), ("power", "Average power reduction (x)")): for suffix, diverging in (("comparison", False), ("diverging", True)): draw_metric(compact_speedups, metric, label, f"{metric}_compact_{suffix}", args.output_dir, diverging=diverging, compact=True) for metric, name, label, mode in [ ("latency_speedup", "longest_stage_latency", "Longest stage latency speedup (x)", None), ("latency_speedup", "latency", "Single-inference latency speedup (x)", "latency"), ("throughput_speedup", "throughput", "Throughput speedup (x)", "throughput"), ("power", "power", "Average power reduction (x)", None), ("energy", "energy", "Energy reduction (x)", None), ]: selected = [row for row in comparisons if mode is None or row.mode == mode] draw_metric(selected, metric, label, f"{name}_comparison", args.output_dir) draw_metric(selected, metric, label, f"{name}_diverging", args.output_dir, diverging=True) for metric, name, label, scale, mode in [ ("latency", "longest_stage_latency", "Longest stage latency (ms)", 1.0, None), ("latency", "latency", "Single-inference latency (ms)", 1.0, "latency"), ("power", "power", "Average power (mW)", 1.0, None), ("energy", "energy", "Energy per inference (mJ)", 1e-9, None), ("throughput", "throughput", "Throughput (inferences/s)", 1.0, "throughput"), ]: for suffix, group, raptor_only in ( ("absolute", comparisons, False), ("yolo", yolo_only, True) ): selected = [row for row in group if mode is None or row.mode == mode] if not selected: continue draw_absolute(selected, metric, label, f"{name}_{suffix}", args.output_dir, scale, raptor_only=raptor_only) if __name__ == "__main__": main()