NiccoloN
a77edafcbf
Validate Operations / validate-operations (push) Has been cancelled
plot adjustments validation adjustments
720 lines
37 KiB
Python
Executable File
720 lines
37 KiB
Python
Executable File
#!/usr/bin/env python3
|
||
"""Generate publication-ready Raptor vs PIMCOMP comparison charts.
|
||
|
||
Usage:
|
||
.venv/bin/python validation/tools/pim/pimcomp/compare/generate_plots.py
|
||
|
||
Dependencies: matplotlib (no pandas or seaborn required).
|
||
By default, reads results_comparison.csv beside this script and writes plots/
|
||
beside it, with outputs grouped in png/, svg/, and pdf/ under the output directory.
|
||
|
||
YOLO models without PIMCOMP measurements get separate Raptor-only charts.
|
||
Comparisons use vertical bars for ratios, diverging advantages, and absolute
|
||
values, with compiler colors and patterns for the Raptor pipeline stages.
|
||
A compact dot-plot summary combines latency, throughput, and both energy cases.
|
||
Throughput-mode latency is the output interval, a bottleneck-stage proxy,
|
||
not end-to-end inference latency. Separate throughput plots use inferences/s.
|
||
Energy is reported per inference; power is averaged over the simulation.
|
||
Numerical results whose compiler did not pass functional validation have a red
|
||
status following their value label; missing PIMCOMP
|
||
measurements appear as 'N/A'. The fixed compiler palette is shared by
|
||
all metrics. Relative plots use PIMCOMP / Raptor for latency, power and energy,
|
||
and Raptor / PIMCOMP for throughput, so >1 favors Raptor. Diverging plots put the winner's
|
||
factor above (Raptor) or below (PIMCOMP) parity, with 1x at the zero line.
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import csv
|
||
import math
|
||
import sys
|
||
from dataclasses import dataclass
|
||
from itertools import groupby
|
||
from pathlib import Path
|
||
|
||
import matplotlib
|
||
matplotlib.use("Agg")
|
||
import matplotlib.pyplot as plt
|
||
from matplotlib.patches import Patch, Rectangle
|
||
from matplotlib.lines import Line2D
|
||
from matplotlib.text import Text
|
||
from matplotlib.ticker import FuncFormatter, MultipleLocator
|
||
from matplotlib.transforms import Bbox
|
||
|
||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||
from plot_style import (ARCH_NAMES, ARCH_STYLES, BAND_COLOR, COMPACT_MODE_NAMES, COMPILER_NAMES, COMPILER_STYLES,
|
||
DAGGER_COLOR, ERROR_COLOR, LINE_COLOR, MATRIX_TEXT_COLOR,
|
||
MODE_NAMES, MODEL_NAMES, NETWORK_STYLES,
|
||
PALETTE, TEXT_COLOR, add_legend, configure_style, fit_value_labels,
|
||
fit_bar_widths, fit_x_tick_labels, save_figure, value_label)
|
||
|
||
|
||
STAGE_PATTERNS = {1: "", 2: "..", 4: "//", 8: "\\\\"}
|
||
ARCHITECTURES = ("arch-a", "arch-b")
|
||
MODELS = ("vgg8", "resnet18", "resnet34", "googlenet", "yolo11n")
|
||
CONFIGURATIONS = (("latency", 1), ("throughput", 2), ("throughput", 4), ("throughput", 8))
|
||
VALIDATION_STYLES = {
|
||
"PASS": PALETTE["sage"],
|
||
"FAIL": PALETTE["red"],
|
||
"SKIPPED": PALETTE["pale_sand"],
|
||
"NA": PALETTE["pale_blue"],
|
||
}
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class Result:
|
||
arch: str
|
||
model: str
|
||
mode: str
|
||
pipeline: int
|
||
pimcomp_schedule: str
|
||
raptor_validation: str
|
||
pimcomp_validation: str
|
||
latency_speedup: float
|
||
power: float
|
||
energy: float
|
||
raptor_latency: float
|
||
pimcomp_latency: float
|
||
raptor_throughput: float
|
||
pimcomp_throughput: float
|
||
raptor_power: float
|
||
pimcomp_power: float
|
||
raptor_energy: float
|
||
pimcomp_energy: float
|
||
|
||
@property
|
||
def comparison_passed(self) -> bool:
|
||
return self.raptor_validation == self.pimcomp_validation == "PASS"
|
||
|
||
@property
|
||
def throughput_speedup(self) -> float:
|
||
return quotient(self.raptor_throughput, self.pimcomp_throughput)
|
||
|
||
@property
|
||
def speedup(self) -> float:
|
||
return self.latency_speedup if self.mode == "latency" else self.throughput_speedup
|
||
|
||
|
||
def positive_float(text: str | None) -> float:
|
||
try:
|
||
value = float(text)
|
||
except (TypeError, ValueError):
|
||
return math.nan
|
||
return value if math.isfinite(value) and value > 0 else math.nan
|
||
|
||
|
||
def quotient(numerator: float, denominator: float) -> float:
|
||
if not (math.isfinite(numerator) and math.isfinite(denominator)):
|
||
return math.nan
|
||
return numerator / denominator
|
||
|
||
|
||
def validation_status(text: str) -> str:
|
||
status = text.strip().upper()
|
||
if status not in VALIDATION_STYLES:
|
||
raise ValueError(f"Unknown functional-validation status: {text!r}")
|
||
return status
|
||
|
||
|
||
def load_results(path: Path) -> list[Result]:
|
||
with path.open(newline="", encoding="utf-8-sig") as handle:
|
||
source = list(csv.DictReader(handle))
|
||
rows = []
|
||
expected = {(arch, model, mode, stage)
|
||
for arch in ARCHITECTURES for model in MODELS
|
||
for mode, stage in CONFIGURATIONS}
|
||
seen = {}
|
||
for line, row in enumerate(source, 2):
|
||
try:
|
||
arch, model, mode = (row[field].strip().lower() for field in ("arch", "model", "mode"))
|
||
stage = int(row["raptor_pipeline"])
|
||
except (KeyError, AttributeError, TypeError, ValueError) as exc:
|
||
raise ValueError(f"Invalid configuration key at CSV row {line}: {exc}") from exc
|
||
key = arch, model, mode, stage
|
||
if key not in expected:
|
||
raise ValueError(f"Unexpected configuration at CSV row {line}: {key}")
|
||
if key in seen:
|
||
raise ValueError(f"Duplicate configuration {key} at CSV rows {seen[key]} and {line}")
|
||
seen[key] = line
|
||
schedule = row["pimcomp_pipeline"].strip().lower()
|
||
if schedule != ("element" if mode == "latency" else "batch"):
|
||
raise ValueError(f"{key}: incompatible PIMCOMP schedule {schedule!r}")
|
||
rows.append(Result(
|
||
arch=arch,
|
||
model=model,
|
||
mode=mode,
|
||
pipeline=stage,
|
||
pimcomp_schedule=schedule,
|
||
raptor_validation=validation_status(row["raptor_functional_validation"]),
|
||
pimcomp_validation=validation_status(row["pimcomp_functional_validation"]),
|
||
latency_speedup=quotient(positive_float(row["pimcomp_latency_ms"]),
|
||
positive_float(row["raptor_latency_ms"])),
|
||
power=quotient(positive_float(row["pimcomp_power_mw"]),
|
||
positive_float(row["raptor_power_mw"])),
|
||
energy=quotient(positive_float(row["pimcomp_energy_pj"]),
|
||
positive_float(row["raptor_energy_pj"])),
|
||
raptor_latency=positive_float(row["raptor_latency_ms"]),
|
||
pimcomp_latency=positive_float(row["pimcomp_latency_ms"]),
|
||
raptor_throughput=positive_float(row["raptor_throughput_samples_s"]),
|
||
pimcomp_throughput=positive_float(row["pimcomp_throughput_samples_s"]),
|
||
raptor_power=positive_float(row["raptor_power_mw"]),
|
||
pimcomp_power=positive_float(row["pimcomp_power_mw"]),
|
||
raptor_energy=positive_float(row["raptor_energy_pj"]),
|
||
pimcomp_energy=positive_float(row["pimcomp_energy_pj"]),
|
||
))
|
||
missing = expected - seen.keys()
|
||
if missing:
|
||
raise ValueError(f"Missing {len(missing)} expected configurations. Examples: {sorted(missing)[:5]}")
|
||
rows.sort(key=lambda r: (ARCHITECTURES.index(r.arch), MODELS.index(r.model), r.pipeline))
|
||
absolute_columns(rows)
|
||
return rows
|
||
|
||
|
||
def advantage_compiler(ratio: float) -> str | None:
|
||
if ratio > 1:
|
||
return "raptor"
|
||
if ratio < 1:
|
||
return "pimcomp"
|
||
return None
|
||
|
||
|
||
def figure_legend(fig, results: list[Result], field: str,
|
||
absolute=False, raptor_only=False):
|
||
compilers = ("raptor",) if raptor_only else ("raptor", "pimcomp")
|
||
if absolute:
|
||
measurements = [(row, compiler, getattr(row, f"{compiler}_{field}"))
|
||
for row, compiler in absolute_columns(results, raptor_only)]
|
||
else:
|
||
measurements = [(row, None, getattr(row, field)) for row in results]
|
||
available = [(row, compiler, value) for row, compiler, value in measurements
|
||
if math.isfinite(value)]
|
||
stages = {row.pipeline for row, compiler, _ in available if compiler != "pimcomp"}
|
||
handles = []
|
||
visible_compilers = {compiler if absolute else advantage_compiler(value)
|
||
for _, compiler, value in available} - {None}
|
||
if len(visible_compilers) > 1:
|
||
for compiler in compilers:
|
||
fill, border = COMPILER_STYLES[compiler]
|
||
handles.append(Patch(facecolor=fill, edgecolor=border,
|
||
label=COMPILER_NAMES[compiler] + ("" if absolute else " advantage")))
|
||
handles += [Patch(facecolor="white", edgecolor=TEXT_COLOR, hatch=STAGE_PATTERNS[depth],
|
||
label=("1 stage (latency)" if absolute else "Single-inference")
|
||
if depth == 1 else f"{depth} stages")
|
||
for depth in sorted(stages) if stages != {1}]
|
||
styles = []
|
||
if len(available) < len(measurements):
|
||
styles.append(Patch(facecolor="white", edgecolor=LINE_COLOR,
|
||
label="N/A: missing measurement"))
|
||
handles += styles
|
||
if handles:
|
||
add_legend(fig, handles)
|
||
|
||
|
||
def absolute_columns(results: list[Result], raptor_only: bool = False) -> list[tuple[Result, str]]:
|
||
columns = []
|
||
for (arch, model), group in groupby(results, key=lambda row: (row.arch, row.model)):
|
||
rows = sorted(group, key=lambda row: row.pipeline)
|
||
for row in rows:
|
||
columns.append((row, "raptor"))
|
||
if row.mode == "latency" and not raptor_only:
|
||
columns.append((row, "pimcomp"))
|
||
throughput = [row for row in rows if row.mode == "throughput"]
|
||
if throughput and not raptor_only:
|
||
signatures = {
|
||
(row.pimcomp_schedule, row.pimcomp_validation,
|
||
*(None if math.isnan(value) else value for value in (
|
||
row.pimcomp_latency, row.pimcomp_throughput, row.pimcomp_power, row.pimcomp_energy)))
|
||
for row in throughput
|
||
}
|
||
if len(signatures) != 1:
|
||
raise ValueError(f"{arch}/{model}: inconsistent shared PIMCOMP batch results across pipelines")
|
||
columns.append((throughput[0], "pimcomp"))
|
||
return columns
|
||
|
||
|
||
def label_groups(ax, results: list[Result], positions: list[float] | None = None) -> None:
|
||
if positions is None:
|
||
positions = list(range(len(results)))
|
||
centers, labels = [], []
|
||
start = 0
|
||
for index, (model, rows) in enumerate(groupby(results, key=lambda row: row.model)):
|
||
end = start + len(list(rows))
|
||
centers.append((positions[start] + positions[end - 1]) / 2)
|
||
labels.append(MODEL_NAMES[model])
|
||
left = ax.get_xlim()[0] if start == 0 else (positions[start - 1] + positions[start]) / 2
|
||
right = ax.get_xlim()[1] if end == len(results) else (positions[end - 1] + positions[end]) / 2
|
||
if index % 2 == 0:
|
||
ax.axvspan(left, right, facecolor=BAND_COLOR, zorder=0)
|
||
if index > 0:
|
||
ax.axvline(left, color=LINE_COLOR, linewidth=0.7)
|
||
start = end
|
||
ax.set_xticks(centers, labels, fontsize=9.5)
|
||
|
||
|
||
def grouped_positions(results: list[Result]) -> list[float]:
|
||
positions = []
|
||
for index, row in enumerate(results):
|
||
if index == 0:
|
||
positions.append(0.0)
|
||
continue
|
||
previous = results[index - 1]
|
||
gap = 0.8 if row.model != previous.model else 0.4 if row.mode != previous.mode else 0.0
|
||
positions.append(positions[-1] + 1.0 + gap)
|
||
return positions
|
||
|
||
|
||
def diverging_ratio(ratio: float) -> float:
|
||
return ratio - 1 if ratio >= 1 else 1 - 1 / ratio
|
||
|
||
|
||
def draw_validation(results: list[Result], output_dir: Path, compact: bool = False) -> None:
|
||
cells = {}
|
||
for row, compiler in absolute_columns(results):
|
||
stage = row.pipeline if compiler == "raptor" else row.pimcomp_schedule
|
||
key = (row.arch, row.model, row.mode, None if compact else stage, compiler)
|
||
cells.setdefault(key, set()).add(getattr(row, f"{compiler}_validation"))
|
||
configurations = list(dict.fromkeys(key[2:] for key in cells))
|
||
positions = []
|
||
for index, (mode, _, _) in enumerate(configurations):
|
||
gap = 0.4 if index and mode != configurations[index - 1][0] else 0
|
||
positions.append(positions[-1] + 1 + gap if index else 0)
|
||
configuration_x = dict(zip(configurations, positions))
|
||
headers = [f"{COMPILER_NAMES['raptor']}\n{stage} stages" if mode == "throughput" and compiler == "raptor" and not compact
|
||
else "PIMCOMP\nbatch" if mode == "throughput" and compiler == "pimcomp" and not compact
|
||
else COMPILER_NAMES[compiler]
|
||
for mode, stage, compiler in configurations]
|
||
fig, axes = plt.subplots(1, 2, figsize=(8.5 if compact else 12, 3.3), layout="constrained")
|
||
for ax, arch in zip(axes, ARCHITECTURES):
|
||
ax.set_xlim(positions[0] - 0.55, positions[-1] + 0.55)
|
||
ax.set_ylim(len(MODELS) - 0.5, -0.5)
|
||
centers, modes = [], []
|
||
for mode, group in groupby(zip(configurations, positions), key=lambda item: item[0][0]):
|
||
group_positions = [x for _, x in group]
|
||
centers.append((group_positions[0] + group_positions[-1]) / 2)
|
||
modes.append(COMPACT_MODE_NAMES[mode] if compact else
|
||
"Latency" if mode == "latency" else MODE_NAMES[mode])
|
||
ax.set_xticks(centers, modes, fontsize=10)
|
||
ax.set_xticks(positions, headers, minor=True)
|
||
ax.tick_params(axis="x", which="minor", top=True, labeltop=True,
|
||
bottom=False, labelbottom=False, length=0, pad=6, labelsize=9)
|
||
ax.set_yticks(range(len(MODELS)), [MODEL_NAMES[model] for model in MODELS], fontsize=9)
|
||
ax.tick_params(axis="both", length=0, pad=6)
|
||
ax.tick_params(axis="y", labelleft=arch == ARCHITECTURES[0])
|
||
architecture = ARCH_NAMES[arch]
|
||
ax.set_xlabel(architecture, fontsize=11, weight="bold", labelpad=8)
|
||
for (cell_arch, model, mode, stage, compiler), statuses in cells.items():
|
||
if cell_arch != arch:
|
||
continue
|
||
x, y = configuration_x[(mode, stage, compiler)], MODELS.index(model)
|
||
status = next(status for status in ("FAIL", "SKIPPED", "NA", "PASS") if status in statuses)
|
||
color, border = VALIDATION_STYLES[status]
|
||
ax.add_patch(Rectangle((x - 0.5, y - 0.42), 1, 0.84,
|
||
facecolor=color, edgecolor=border, linewidth=0.7))
|
||
ax.text(x, y, "N/A" if status == "NA" else status,
|
||
color=MATRIX_TEXT_COLOR, ha="center", va="center", fontsize=12, weight="normal")
|
||
fit_x_tick_labels(fig)
|
||
save_figure(fig, "validation_status_compact" if compact else "validation_status", output_dir)
|
||
|
||
|
||
def select_compact_speedups(results: list[Result]) -> list[Result]:
|
||
selected = []
|
||
for _, group in groupby(results, key=lambda row: (row.arch, row.model, row.mode)):
|
||
rows = list(group)
|
||
selected.append(max(rows, key=lambda row: row.speedup if math.isfinite(row.speedup) else -math.inf))
|
||
return selected
|
||
|
||
|
||
def draw_compact_summary(results: list[Result], output_dir: Path, *, marker_borders: bool = True) -> None:
|
||
selected = select_compact_speedups([row for row in results if row.model != "yolo11n"])
|
||
lookup = {(row.model, row.arch, row.mode): row for row in selected}
|
||
keys = [(model, arch) for model in MODELS if model != "yolo11n" for arch in ARCHITECTURES]
|
||
panels = ((f"{COMPACT_MODE_NAMES['latency']}\nspeedup", "latency", "latency_speedup"),
|
||
(f"{COMPACT_MODE_NAMES['throughput']}\nspeedup", "throughput", "throughput_speedup"),
|
||
(f"{COMPACT_MODE_NAMES['latency']}\nenergy reduction", "latency", "energy"),
|
||
(f"{COMPACT_MODE_NAMES['throughput']}\nenergy reduction", "throughput", "energy"))
|
||
measurements = []
|
||
for _, mode, field in panels:
|
||
points = []
|
||
for model, arch in keys:
|
||
row = lookup[(model, arch, mode)]
|
||
ratio = getattr(row, field)
|
||
if not math.isfinite(ratio) or ratio <= 0:
|
||
raise ValueError(f"Compact summary requires a positive {field} ratio for {arch}/{model}")
|
||
pimcomp_invalid = row.pimcomp_schedule == "batch" or row.pimcomp_validation == "FAIL"
|
||
status = ("P. INVALID" if pimcomp_invalid else
|
||
f"{COMPILER_NAMES['raptor'][0]}. INVALID" if row.raptor_validation == "FAIL" else
|
||
"NOT VALIDATED" if not row.comparison_passed else "")
|
||
points.append((row, ratio, status))
|
||
measurements.append(points)
|
||
ratios = [ratio for points in measurements for _, ratio, _ in points]
|
||
padding = max(max(ratios) - min(ratios), 1) * 0.04
|
||
limits = (0.25 if min(ratios) < 0.5 else 0.5, max(1, *ratios) + padding)
|
||
fig, axes = plt.subplots(4, 1, sharex=True, figsize=(6.8, 7.6), layout="constrained")
|
||
markers = dict(zip((model for model in MODELS if model != "yolo11n"), ("o", "s", "^", "D")))
|
||
lanes = {arch: index * 2 for index, arch in enumerate(ARCHITECTURES)}
|
||
daggers = {}
|
||
for ax, (title, _, _), points in zip(axes, panels, measurements):
|
||
ax.set_xlim(*limits)
|
||
ax.set_ylim(4, -2)
|
||
ax.set_ylabel(title, rotation=90, ha="center", va="bottom", multialignment="center",
|
||
fontsize=8.5, labelpad=10)
|
||
ax.set_yticks(list(lanes.values()), [ARCH_NAMES[arch] for arch in lanes], fontsize=8)
|
||
ax.set_xticks([limits[0], *range(1, math.floor(limits[1]) + 1)])
|
||
ax.xaxis.set_major_formatter(FuncFormatter(lambda value, _: f"{value:g}"))
|
||
ax.tick_params(axis="both", length=0, pad=4, labelsize=8.5)
|
||
ax.tick_params(axis="x", bottom=False, labelbottom=False,
|
||
top=ax is axes[0], labeltop=ax is axes[0])
|
||
ax.grid(axis="x", linewidth=0.6)
|
||
ax.axvline(1, color=TEXT_COLOR, linewidth=0.9, linestyle="--", zorder=1)
|
||
for lane in lanes.values():
|
||
ax.axhline(lane, color=LINE_COLOR, linewidth=0.6)
|
||
ax.spines["left"].set_visible(False)
|
||
ax.spines["bottom"].set_visible(False)
|
||
ax.spines["top"].set_visible(ax is axes[0])
|
||
for row, ratio, status in points:
|
||
fill, border = NETWORK_STYLES[row.model]
|
||
lane = lanes[row.arch]
|
||
ax.plot(ratio, lane, marker=markers[row.model], markersize=6.5, linestyle="none",
|
||
markerfacecolor=fill if marker_borders else border, markeredgecolor=border,
|
||
markeredgewidth=1.1 if marker_borders else 0)
|
||
suffix = f" [{status}]" if status and status != "P. INVALID" else ""
|
||
label = f"{ratio:.2f}×{suffix}"
|
||
annotation = ax.annotate(label, (ratio, lane), xytext=(0, -5), textcoords="offset points",
|
||
rotation=90, rotation_mode="anchor", ha="right", va="center",
|
||
fontsize=7.5, color=TEXT_COLOR, math_fontfamily="cm")
|
||
annotation.set_in_layout(False)
|
||
dagger = ax.annotate(r"$\boldsymbol{\dagger}$", (0.5, 1), xycoords=annotation,
|
||
xytext=(0, 1), textcoords="offset points", rotation=90,
|
||
rotation_mode="anchor", ha="left", va="center", fontsize=9,
|
||
color=DAGGER_COLOR, math_fontfamily="cm",
|
||
alpha=1 if status == "P. INVALID" else 0)
|
||
dagger.set_in_layout(False)
|
||
daggers[annotation] = dagger
|
||
handles = [Line2D([], [], marker=marker, color=NETWORK_STYLES[model][1],
|
||
markerfacecolor=NETWORK_STYLES[model][0 if marker_borders else 1], markersize=6.5,
|
||
markeredgewidth=1 if marker_borders else 0,
|
||
linestyle="none", label=MODEL_NAMES[model])
|
||
for model, marker in markers.items()]
|
||
handles.append(Text(0, 0, r"$\boldsymbol{\dagger}$", color=DAGGER_COLOR, fontsize=9,
|
||
math_fontfamily="cm", label="PIMCOMP invalid"))
|
||
add_legend(fig, handles)
|
||
axes[0].xaxis.set_label_position("top")
|
||
axes[0].set_xlabel("Ratio (x)", fontsize=9, labelpad=7, va="bottom")
|
||
axes[0].xaxis.set_label_coords(0.5, 1.18)
|
||
axes[0].text(0, 1.18, "← PIMCOMP better", transform=axes[0].transAxes, ha="left", va="bottom", fontsize=8)
|
||
axes[0].text(1, 1.18, f"{COMPILER_NAMES['raptor']} better →", transform=axes[0].transAxes,
|
||
ha="right", va="bottom", fontsize=8)
|
||
fig.canvas.draw()
|
||
renderer = fig.canvas.get_renderer()
|
||
|
||
def label_bounds(text):
|
||
return Bbox.union([text.get_window_extent(renderer), daggers[text].get_window_extent(renderer)])
|
||
|
||
def position_label(text, dx, dy):
|
||
text.set_ha("left" if dy > 0 else "right")
|
||
extra = (daggers[text].get_window_extent(renderer).height + renderer.points_to_pixels(1)
|
||
if dy < 0 else 0)
|
||
text.set_position((dx, dy - extra * 72 / fig.dpi))
|
||
|
||
labels_by_axis = [[text for text in ax.texts if getattr(text, "xycoords", None) == "data"] for ax in axes]
|
||
label_height = max(label_bounds(text).height for labels in labels_by_axis for text in labels)
|
||
lane_gap = min(abs(ax.transData.transform((0, 0))[1] - ax.transData.transform((0, 2))[1])
|
||
for ax in axes)
|
||
height_scale = (label_height + renderer.points_to_pixels(5)) / lane_gap
|
||
if height_scale > 1:
|
||
fig.set_size_inches(fig.get_figwidth(), fig.get_figheight() * height_scale)
|
||
fig.canvas.draw()
|
||
renderer = fig.canvas.get_renderer()
|
||
for ax, labels in zip(axes, labels_by_axis):
|
||
placed = []
|
||
for line in ax.lines:
|
||
placed.append(line.get_window_extent(renderer).padded(1))
|
||
bounds = ax.get_window_extent(renderer)
|
||
choices = []
|
||
for text in labels:
|
||
preferred_dy = 5 if text.xy[1] == lanes[ARCHITECTURES[0]] else -5
|
||
candidates = []
|
||
for dx in (0, -3, 3, -6, 6, -8, 8):
|
||
for dy in (-5, 5):
|
||
position_label(text, dx, dy)
|
||
box = label_bounds(text)
|
||
overlap = any(box.overlaps(other) for other in placed)
|
||
outside = not (bounds.x0 <= box.x0 <= box.x1 <= bounds.x1 and
|
||
bounds.y0 <= box.y0 <= box.y1 <= bounds.y1)
|
||
if not outside and not overlap:
|
||
candidates.append(((dy != preferred_dy, abs(dx) + abs(dy)), dx, dy, box))
|
||
choices.append((text, candidates))
|
||
layouts = [((0, 0), [])]
|
||
for text, candidates in sorted(choices, key=lambda choice: len(choice[1])):
|
||
layouts = sorted(
|
||
(((cost[0] + distance[0], cost[1] + distance[1]), assignments + [(text, dx, dy, box)])
|
||
for cost, assignments in layouts for distance, dx, dy, box in candidates
|
||
if not any(box.overlaps(other[3]) for other in assignments)),
|
||
key=lambda layout: layout[0])[:32]
|
||
if not layouts:
|
||
raise ValueError("Dot labels do not fit without overlap; increase lane spacing or figure width")
|
||
for text, dx, dy, _ in layouts[0][1]:
|
||
position_label(text, dx, dy)
|
||
save_figure(fig, "performance_energy_compact", output_dir)
|
||
|
||
|
||
def draw_metric(results: list[Result], field: str, ylabel: str,
|
||
basename: str, output_dir: Path, diverging: bool = False,
|
||
compact: bool = False) -> None:
|
||
data = [diverging_ratio(getattr(row, field)) if diverging else getattr(row, field)
|
||
for row in results]
|
||
available = [v for v in data if math.isfinite(v)]
|
||
upper = max(0, max(available, default=1.0))
|
||
lower = (min(0, min(available, default=0)) if diverging else
|
||
0.25 if min(available, default=1) < 0.5 else 0.5)
|
||
padding = max(upper - lower, 1.0) * 0.3
|
||
y_max = math.ceil((upper + padding) * 2) / 2
|
||
y_min = math.floor((lower - padding) * 2) / 2 if diverging else lower
|
||
y_interval = 1.0 if y_max - y_min > 4.5 else 0.5
|
||
|
||
fig, axes = plt.subplots(1, 2, sharey=True, figsize=(8.5 if compact else 12, 4.4),
|
||
layout="constrained")
|
||
if compact:
|
||
metric_suffix = "" if field == "speedup" else f" {field}"
|
||
panel_labels = {mode: label + metric_suffix for mode, label in COMPACT_MODE_NAMES.items()}
|
||
handles = [Patch(facecolor=fill, edgecolor=border, hatch=pattern, label=ARCH_NAMES[arch])
|
||
for arch, (fill, border, pattern) in ARCH_STYLES.items()]
|
||
|
||
panels = ("latency", "throughput") if compact else ARCHITECTURES
|
||
for panel_index, (ax, panel) in enumerate(zip(axes, panels)):
|
||
selected = [row for row in results if getattr(row, "mode" if compact else "arch") == panel]
|
||
if compact:
|
||
selected.sort(key=lambda row: (MODELS.index(row.model), ARCHITECTURES.index(row.arch)))
|
||
positions = [index + index // 2 for index in range(len(selected))] if compact else grouped_positions(selected)
|
||
ax.set_xlabel(panel_labels[panel] if compact else ARCH_NAMES[panel],
|
||
fontsize=11, weight="bold", labelpad=8)
|
||
ax.set_axisbelow(True)
|
||
ax.set_ylim(y_min, y_max)
|
||
ax.set_xlim(-0.9, positions[-1] + 0.9 if positions else 0.9)
|
||
label_groups(ax, selected, positions)
|
||
ax.tick_params(axis="y", labelleft=panel_index == 0, pad=5, length=0)
|
||
ax.tick_params(axis="x", labelsize=9.5, length=0, pad=6)
|
||
ax.yaxis.set_major_locator(MultipleLocator(y_interval))
|
||
if diverging:
|
||
ax.yaxis.set_major_formatter(FuncFormatter(lambda value, _: f"{abs(value) + 1:g}"))
|
||
else:
|
||
ax.set_yticks([y_min], minor=True)
|
||
ax.yaxis.set_minor_formatter(FuncFormatter(lambda value, _: f"{value:g}"))
|
||
ax.tick_params(axis="y", which="minor", labelleft=panel_index == 0, pad=5, length=0)
|
||
ax.grid(axis="y")
|
||
ax.axhline(0 if diverging else 1, color=TEXT_COLOR, linewidth=1.0,
|
||
linestyle="--", zorder=3)
|
||
if panel_index == 0:
|
||
ax.set_ylabel(ylabel, fontsize=10.5, labelpad=6)
|
||
|
||
for position, row in zip(positions, selected):
|
||
ratio = getattr(row, field)
|
||
if not math.isfinite(ratio):
|
||
ax.text(position, (0 if diverging else y_min) + (y_max - y_min) * 0.02, "N/A", ha="center", va="bottom",
|
||
fontsize=9.5, color=TEXT_COLOR, fontstyle="italic")
|
||
continue
|
||
compiler = advantage_compiler(ratio)
|
||
statuses = [status for status in (row.raptor_validation, row.pimcomp_validation) if status != "PASS"]
|
||
status = "FAIL" if "FAIL" in statuses else next(iter(statuses), "PASS")
|
||
pattern = STAGE_PATTERNS[row.pipeline]
|
||
color, border = COMPILER_STYLES[compiler] if compiler else ("white", TEXT_COLOR)
|
||
if compact:
|
||
color, border, pattern = ARCH_STYLES[row.arch]
|
||
measurement = diverging_ratio(ratio) if diverging else ratio
|
||
ax.bar(position, measurement, width=1.0,
|
||
color=color,
|
||
hatch=pattern,
|
||
edgecolor=border, linewidth=0.7, zorder=2)
|
||
direction = -1 if measurement < 0 else 1
|
||
factor = max(ratio, 1 / ratio) if diverging else ratio
|
||
value_label(ax, position, measurement + direction * (y_max - y_min) * 0.012,
|
||
f"{factor:.2f}×", status, direction, fontsize=9.1, invalid_label="P. INVALID")
|
||
|
||
fit_bar_widths(fig)
|
||
if compact:
|
||
add_legend(fig, handles)
|
||
else:
|
||
figure_legend(fig, results, field)
|
||
fit_x_tick_labels(fig)
|
||
fit_value_labels(fig, lower_limit=None if diverging else y_min)
|
||
save_figure(fig, basename, output_dir)
|
||
|
||
|
||
def draw_absolute(results: list[Result], field: str, ylabel: str,
|
||
basename: str, output_dir: Path, scale: float = 1.0,
|
||
raptor_only: bool = False) -> None:
|
||
columns = absolute_columns(results, raptor_only)
|
||
available = [getattr(row, f"{compiler}_{field}") * scale
|
||
for row, compiler in columns
|
||
if math.isfinite(getattr(row, f"{compiler}_{field}"))]
|
||
y_max = max(available, default=1.0) * 1.35
|
||
fig, axes = plt.subplots(1, 2, sharey=True, figsize=(12, 4.4),
|
||
layout="constrained")
|
||
|
||
for ax, arch in zip(axes, ARCHITECTURES):
|
||
selected = [(row, compiler) for row, compiler in columns if row.arch == arch]
|
||
positions = grouped_positions([row for row, _ in selected])
|
||
ax.set_xlabel(ARCH_NAMES[arch],
|
||
fontsize=11, weight="bold", labelpad=8)
|
||
ax.set_axisbelow(True)
|
||
ax.set_ylim(0, y_max)
|
||
if positions:
|
||
ax.set_xlim(positions[0] - 0.9, positions[-1] + 0.9)
|
||
else:
|
||
ax.set_xlim(-0.9, 0.9)
|
||
label_groups(ax, [row for row, _ in selected], positions)
|
||
if {row.mode for row, _ in selected} == {"latency", "throughput"}:
|
||
centers, labels = [], []
|
||
for (_, mode), group in groupby(enumerate(selected), key=lambda item: (item[1][0].model, item[1][0].mode)):
|
||
group_positions = [positions[position] for position, _ in group]
|
||
centers.append((group_positions[0] + group_positions[-1]) / 2)
|
||
labels.append("Latency" if mode == "latency" else MODE_NAMES[mode])
|
||
ax.set_xticks(centers, labels, minor=True)
|
||
ax.tick_params(axis="x", which="minor", length=0, pad=4, labelsize=8)
|
||
if arch == ARCHITECTURES[0]:
|
||
ax.set_ylabel(ylabel, fontsize=10.5)
|
||
if field == "throughput":
|
||
ax.ticklabel_format(axis="y", style="plain", useOffset=False)
|
||
ax.grid(axis="y")
|
||
ax.tick_params(axis="y", labelleft=arch == ARCHITECTURES[0], length=0, pad=6)
|
||
ax.tick_params(axis="x", which="major", length=0,
|
||
pad=22 if {row.mode for row, _ in selected} == {"latency", "throughput"} else 6)
|
||
|
||
for x, (row, compiler) in zip(positions, selected):
|
||
measurement = getattr(row, f"{compiler}_{field}") * scale
|
||
if not math.isfinite(measurement):
|
||
ax.text(x, y_max * 0.02, "N/A", rotation=90, ha="center",
|
||
va="bottom", fontsize=8, color=TEXT_COLOR)
|
||
continue
|
||
status = getattr(row, f"{compiler}_validation")
|
||
pattern = STAGE_PATTERNS[row.pipeline] if compiler == "raptor" else ""
|
||
fill, border = COMPILER_STYLES[compiler]
|
||
ax.bar(x, measurement, width=1.0,
|
||
color=fill,
|
||
hatch=pattern,
|
||
edgecolor=border, linewidth=0.7, zorder=2)
|
||
label = (f"{measurement:,.2f}".rstrip("0").rstrip(".")
|
||
if field == "throughput" else f"{measurement:.3g}")
|
||
value_label(ax, x, measurement + y_max * 0.012, label, status, invalid_label="P. INVALID")
|
||
|
||
fit_bar_widths(fig)
|
||
figure_legend(fig, results, field, absolute=True, raptor_only=raptor_only)
|
||
fit_x_tick_labels(fig)
|
||
fit_value_labels(fig)
|
||
save_figure(fig, basename, output_dir)
|
||
|
||
|
||
def draw_pipeline_effect(results: list[Result], output_dir: Path, *, marker_borders: bool = True) -> None:
|
||
rows = [row for row in results if row.mode == "throughput" or row.pipeline == 1]
|
||
stages = sorted({row.pipeline for row in rows})
|
||
panels = [model for model in MODELS if any(row.model == model for row in rows)]
|
||
lookup = {(row.arch, row.model, row.pipeline): row for row in rows}
|
||
rates = {key: quotient(1000, row.raptor_latency) if row.mode == "latency" else row.raptor_throughput
|
||
for key, row in lookup.items()}
|
||
fig, axes = plt.subplots(len(ARCHITECTURES), len(panels), sharex=True, sharey="col",
|
||
figsize=(2.6 * len(panels), 4.3), squeeze=False, layout="constrained")
|
||
fig.supylabel("Throughput (inferences/s)", fontsize=10.5)
|
||
for column, model in enumerate(panels):
|
||
fill, border = NETWORK_STYLES[model]
|
||
for index, arch in enumerate(ARCHITECTURES):
|
||
ax = axes[index, column]
|
||
keys = [(arch, model, stage) for stage in stages]
|
||
values = [rates.get(key, math.nan) for key in keys]
|
||
ax.plot(stages, values, color=border, linewidth=1.8)
|
||
for stage, value in zip(stages, values):
|
||
if math.isfinite(value):
|
||
ax.plot(stage, value, color=border,
|
||
markerfacecolor=fill if marker_borders else border, marker="s" if stage == 1 else "o",
|
||
markeredgewidth=1 if marker_borders else 0, markersize=7, linestyle="none")
|
||
ax.set_xticks(stages)
|
||
if index == 0:
|
||
ax.set_title(MODEL_NAMES[model], fontsize=10.5, weight="normal", pad=8)
|
||
ax.grid(axis="y")
|
||
ax.tick_params(axis="both", length=0, pad=6,
|
||
labelbottom=index == len(ARCHITECTURES) - 1, labelleft=True)
|
||
ax.ticklabel_format(axis="y", style="plain", useOffset=False)
|
||
if column == len(panels) - 1:
|
||
ax.yaxis.set_label_position("right")
|
||
ax.set_ylabel(ARCH_NAMES[arch], fontsize=10.5, weight="bold", rotation=270, labelpad=24)
|
||
for ax in axes.flat:
|
||
ax.margins(y=0.08)
|
||
handles = [Line2D([], [], color=TEXT_COLOR, markerfacecolor="white" if marker_borders else TEXT_COLOR,
|
||
markeredgewidth=1 if marker_borders else 0,
|
||
marker=marker, markersize=7, linestyle="none", label=label)
|
||
for marker, label in (("s", "Derived as 1 / latency"),
|
||
("o", "Throughput measurement"))]
|
||
add_legend(fig, handles)
|
||
fig.supxlabel(f"{COMPILER_NAMES['raptor']} pipeline stages", fontsize=11)
|
||
fit_x_tick_labels(fig)
|
||
save_figure(fig, "raptor_pipeline_throughput", output_dir)
|
||
|
||
|
||
def main() -> None:
|
||
parser = argparse.ArgumentParser(description=__doc__)
|
||
script_dir = Path(__file__).resolve().parent
|
||
parser.add_argument("--input", type=Path, default=script_dir / "results_comparison.csv")
|
||
parser.add_argument("--output-dir", type=Path, default=script_dir / "plots/default")
|
||
parser.add_argument("--colors", action=argparse.BooleanOptionalAction, default=True,
|
||
help="Use the pastel palette (default); --no-colors exports grayscale charts")
|
||
parser.add_argument("--anonymize", action=argparse.BooleanOptionalAction, default=False,
|
||
help="Replace the displayed Raptor name with Anonimized")
|
||
parser.add_argument("--marker-borders", action=argparse.BooleanOptionalAction, default=True,
|
||
help="Outline markers (default); --no-marker-borders uses solid markers in dot and line charts")
|
||
args = parser.parse_args()
|
||
configure_style(colors=args.colors, anonymize=args.anonymize)
|
||
results = load_results(args.input)
|
||
valid = sum(row.comparison_passed for row in results)
|
||
print(f"Results: {len(results)} configurations, {valid} valid comparisons, "
|
||
f"{len(results)-valid} comparisons without two PASS statuses")
|
||
draw_validation(results, args.output_dir)
|
||
draw_validation(results, args.output_dir, compact=True)
|
||
draw_pipeline_effect(results, args.output_dir, marker_borders=args.marker_borders)
|
||
yolo_only_models = {
|
||
model for model in MODELS if model.startswith("yolo")
|
||
and not any(math.isfinite(getattr(row, f"pimcomp_{field}"))
|
||
for row in results if row.model == model
|
||
for field in ("latency", "power", "energy"))
|
||
}
|
||
comparisons = [row for row in results if row.model not in yolo_only_models]
|
||
yolo_only = [row for row in results if row.model in yolo_only_models]
|
||
compact_speedups = select_compact_speedups(comparisons)
|
||
draw_compact_summary(comparisons, args.output_dir, marker_borders=args.marker_borders)
|
||
for metric, label in (("speedup", "Speedup (x)"),
|
||
("energy", "Energy reduction (x)"),
|
||
("power", "Average power reduction (x)")):
|
||
for suffix, diverging in (("comparison", False), ("diverging", True)):
|
||
draw_metric(compact_speedups, metric, label,
|
||
f"{metric}_compact_{suffix}", args.output_dir, diverging=diverging, compact=True)
|
||
for metric, name, label, mode in [
|
||
("latency_speedup", "longest_stage_latency", "Longest stage latency speedup (x)", None),
|
||
("latency_speedup", "latency", "Single-inference latency speedup (x)", "latency"),
|
||
("throughput_speedup", "throughput", "Throughput speedup (x)", "throughput"),
|
||
("power", "power", "Average power reduction (x)", None),
|
||
("energy", "energy", "Energy reduction (x)", None),
|
||
]:
|
||
selected = [row for row in comparisons if mode is None or row.mode == mode]
|
||
draw_metric(selected, metric, label,
|
||
f"{name}_comparison", args.output_dir)
|
||
draw_metric(selected, metric, label,
|
||
f"{name}_diverging", args.output_dir, diverging=True)
|
||
for metric, name, label, scale, mode in [
|
||
("latency", "longest_stage_latency", "Longest stage latency (ms)", 1.0, None),
|
||
("latency", "latency", "Single-inference latency (ms)", 1.0, "latency"),
|
||
("power", "power", "Average power (mW)", 1.0, None),
|
||
("energy", "energy", "Energy per inference (mJ)", 1e-9, None),
|
||
("throughput", "throughput", "Throughput (inferences/s)", 1.0, "throughput"),
|
||
]:
|
||
for suffix, group, raptor_only in (
|
||
("absolute", comparisons, False), ("yolo", yolo_only, True)
|
||
):
|
||
selected = [row for row in group if mode is None or row.mode == mode]
|
||
if not selected:
|
||
continue
|
||
draw_absolute(selected, metric, label,
|
||
f"{name}_{suffix}", args.output_dir, scale,
|
||
raptor_only=raptor_only)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|