NiccoloN
577 lines
28 KiB
Python
Executable File
577 lines
28 KiB
Python
Executable File
#!/usr/bin/env python3
|
||
"""Generate publication-ready Raptor vs PIMCOMP comparison charts.
|
||
|
||
Usage:
|
||
.venv/bin/python validation/tools/pim/pimcomp/compare/generate_plots.py
|
||
|
||
Dependencies: matplotlib (no pandas or seaborn required).
|
||
By default, reads results_comparison.csv beside this script and writes plots/
|
||
beside it, with outputs grouped in png/, svg/, and pdf/ under the output directory.
|
||
|
||
YOLO models without PIMCOMP measurements get separate Raptor-only charts.
|
||
Comparisons use vertical bars for ratios, diverging advantages, and absolute
|
||
values, with compiler colors and patterns for the Raptor pipeline stages.
|
||
Throughput-mode latency is the output interval, a bottleneck-stage proxy,
|
||
not end-to-end inference latency. Separate throughput plots use inferences/s.
|
||
Energy is reported per inference; power is averaged over the simulation.
|
||
Numerical results whose compiler did not pass functional validation have a red
|
||
status following their value label; missing PIMCOMP
|
||
measurements appear as 'N/A'. The fixed compiler palette is shared by
|
||
all metrics. Relative plots use PIMCOMP / Raptor for latency, power and energy,
|
||
and Raptor / PIMCOMP for throughput, so >1 favors Raptor. Diverging plots put the winner's
|
||
factor above (Raptor) or below (PIMCOMP) parity, with 1x at the zero line.
|
||
"""
|
||
from __future__ import annotations
|
||
|
||
import argparse
|
||
import csv
|
||
import math
|
||
import sys
|
||
from dataclasses import dataclass
|
||
from itertools import groupby
|
||
from pathlib import Path
|
||
|
||
import matplotlib
|
||
matplotlib.use("Agg")
|
||
import matplotlib.pyplot as plt
|
||
from matplotlib.patches import Patch, Rectangle
|
||
from matplotlib.lines import Line2D
|
||
from matplotlib.ticker import FuncFormatter, MultipleLocator
|
||
|
||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||
from plot_style import (ARCH_NAMES, BAND_COLOR, COMPILER_NAMES, COMPILER_STYLES,
|
||
ERROR_COLOR, LINE_COLOR, MATRIX_TEXT_COLOR, METRIC_STYLES,
|
||
MODE_NAMES, MODEL_NAMES, NETWORK_STYLES,
|
||
PALETTE, TEXT_COLOR, add_legend, configure_style, fit_value_labels,
|
||
fit_bar_widths, fit_x_tick_labels, save_figure, value_label)
|
||
|
||
|
||
STAGE_PATTERNS = {1: "", 2: "..", 4: "//", 8: "\\\\"}
|
||
ARCHITECTURES = ("arch-a", "arch-b")
|
||
MODELS = ("vgg8", "resnet18", "resnet34", "googlenet", "yolo11n")
|
||
CONFIGURATIONS = (("latency", 1), ("throughput", 2), ("throughput", 4), ("throughput", 8))
|
||
VALIDATION_STYLES = {
|
||
"PASS": PALETTE["sage"],
|
||
"FAIL": PALETTE["red"],
|
||
"SKIPPED": PALETTE["pale_sand"],
|
||
"NA": PALETTE["pale_blue"],
|
||
}
|
||
|
||
|
||
@dataclass(frozen=True)
|
||
class Result:
|
||
arch: str
|
||
model: str
|
||
mode: str
|
||
pipeline: int
|
||
pimcomp_schedule: str
|
||
raptor_validation: str
|
||
pimcomp_validation: str
|
||
latency_speedup: float
|
||
power: float
|
||
energy: float
|
||
raptor_latency: float
|
||
pimcomp_latency: float
|
||
raptor_throughput: float
|
||
pimcomp_throughput: float
|
||
raptor_power: float
|
||
pimcomp_power: float
|
||
raptor_energy: float
|
||
pimcomp_energy: float
|
||
|
||
@property
|
||
def comparison_passed(self) -> bool:
|
||
return self.raptor_validation == self.pimcomp_validation == "PASS"
|
||
|
||
@property
|
||
def throughput_speedup(self) -> float:
|
||
return quotient(self.raptor_throughput, self.pimcomp_throughput)
|
||
|
||
@property
|
||
def speedup(self) -> float:
|
||
return self.latency_speedup if self.mode == "latency" else self.throughput_speedup
|
||
|
||
|
||
def positive_float(text: str | None) -> float:
|
||
try:
|
||
value = float(text)
|
||
except (TypeError, ValueError):
|
||
return math.nan
|
||
return value if math.isfinite(value) and value > 0 else math.nan
|
||
|
||
|
||
def quotient(numerator: float, denominator: float) -> float:
|
||
if not (math.isfinite(numerator) and math.isfinite(denominator)):
|
||
return math.nan
|
||
return numerator / denominator
|
||
|
||
|
||
def validation_status(text: str) -> str:
|
||
status = text.strip().upper()
|
||
if status not in VALIDATION_STYLES:
|
||
raise ValueError(f"Unknown functional-validation status: {text!r}")
|
||
return status
|
||
|
||
|
||
def load_results(path: Path) -> list[Result]:
|
||
with path.open(newline="", encoding="utf-8-sig") as handle:
|
||
source = list(csv.DictReader(handle))
|
||
rows = []
|
||
expected = {(arch, model, mode, stage)
|
||
for arch in ARCHITECTURES for model in MODELS
|
||
for mode, stage in CONFIGURATIONS}
|
||
seen = {}
|
||
for line, row in enumerate(source, 2):
|
||
try:
|
||
arch, model, mode = (row[field].strip().lower() for field in ("arch", "model", "mode"))
|
||
stage = int(row["raptor_pipeline"])
|
||
except (KeyError, AttributeError, TypeError, ValueError) as exc:
|
||
raise ValueError(f"Invalid configuration key at CSV row {line}: {exc}") from exc
|
||
key = arch, model, mode, stage
|
||
if key not in expected:
|
||
raise ValueError(f"Unexpected configuration at CSV row {line}: {key}")
|
||
if key in seen:
|
||
raise ValueError(f"Duplicate configuration {key} at CSV rows {seen[key]} and {line}")
|
||
seen[key] = line
|
||
schedule = row["pimcomp_pipeline"].strip().lower()
|
||
if schedule != ("element" if mode == "latency" else "batch"):
|
||
raise ValueError(f"{key}: incompatible PIMCOMP schedule {schedule!r}")
|
||
rows.append(Result(
|
||
arch=arch,
|
||
model=model,
|
||
mode=mode,
|
||
pipeline=stage,
|
||
pimcomp_schedule=schedule,
|
||
raptor_validation=validation_status(row["raptor_functional_validation"]),
|
||
pimcomp_validation=validation_status(row["pimcomp_functional_validation"]),
|
||
latency_speedup=quotient(positive_float(row["pimcomp_latency_ms"]),
|
||
positive_float(row["raptor_latency_ms"])),
|
||
power=quotient(positive_float(row["pimcomp_power_mw"]),
|
||
positive_float(row["raptor_power_mw"])),
|
||
energy=quotient(positive_float(row["pimcomp_energy_pj"]),
|
||
positive_float(row["raptor_energy_pj"])),
|
||
raptor_latency=positive_float(row["raptor_latency_ms"]),
|
||
pimcomp_latency=positive_float(row["pimcomp_latency_ms"]),
|
||
raptor_throughput=positive_float(row["raptor_throughput_samples_s"]),
|
||
pimcomp_throughput=positive_float(row["pimcomp_throughput_samples_s"]),
|
||
raptor_power=positive_float(row["raptor_power_mw"]),
|
||
pimcomp_power=positive_float(row["pimcomp_power_mw"]),
|
||
raptor_energy=positive_float(row["raptor_energy_pj"]),
|
||
pimcomp_energy=positive_float(row["pimcomp_energy_pj"]),
|
||
))
|
||
missing = expected - seen.keys()
|
||
if missing:
|
||
raise ValueError(f"Missing {len(missing)} expected configurations. Examples: {sorted(missing)[:5]}")
|
||
rows.sort(key=lambda r: (ARCHITECTURES.index(r.arch), MODELS.index(r.model), r.pipeline))
|
||
absolute_columns(rows)
|
||
return rows
|
||
|
||
|
||
def advantage_compiler(ratio: float) -> str | None:
|
||
if ratio > 1:
|
||
return "raptor"
|
||
if ratio < 1:
|
||
return "pimcomp"
|
||
return None
|
||
|
||
|
||
def figure_legend(fig, results: list[Result], field: str,
|
||
absolute=False, raptor_only=False):
|
||
compilers = ("raptor",) if raptor_only else ("raptor", "pimcomp")
|
||
if absolute:
|
||
measurements = [(row, compiler, getattr(row, f"{compiler}_{field}"))
|
||
for row, compiler in absolute_columns(results, raptor_only)]
|
||
else:
|
||
measurements = [(row, None, getattr(row, field)) for row in results]
|
||
available = [(row, compiler, value) for row, compiler, value in measurements
|
||
if math.isfinite(value)]
|
||
stages = {row.pipeline for row, compiler, _ in available if compiler != "pimcomp"}
|
||
handles = []
|
||
visible_compilers = {compiler if absolute else advantage_compiler(value)
|
||
for _, compiler, value in available} - {None}
|
||
if len(visible_compilers) > 1:
|
||
for compiler in compilers:
|
||
fill, border = COMPILER_STYLES[compiler]
|
||
handles.append(Patch(facecolor=fill, edgecolor=border,
|
||
label=COMPILER_NAMES[compiler] + ("" if absolute else " advantage")))
|
||
handles += [Patch(facecolor="white", edgecolor=TEXT_COLOR, hatch=STAGE_PATTERNS[depth],
|
||
label=("1 stage (latency)" if absolute else "Single-inference")
|
||
if depth == 1 else f"{depth} stages")
|
||
for depth in sorted(stages) if stages != {1}]
|
||
styles = []
|
||
if len(available) < len(measurements):
|
||
styles.append(Patch(facecolor="white", edgecolor=LINE_COLOR,
|
||
label="N/A: missing measurement"))
|
||
handles += styles
|
||
if handles:
|
||
add_legend(fig, handles)
|
||
|
||
|
||
def absolute_columns(results: list[Result], raptor_only: bool = False) -> list[tuple[Result, str]]:
|
||
columns = []
|
||
for (arch, model), group in groupby(results, key=lambda row: (row.arch, row.model)):
|
||
rows = sorted(group, key=lambda row: row.pipeline)
|
||
for row in rows:
|
||
columns.append((row, "raptor"))
|
||
if row.mode == "latency" and not raptor_only:
|
||
columns.append((row, "pimcomp"))
|
||
throughput = [row for row in rows if row.mode == "throughput"]
|
||
if throughput and not raptor_only:
|
||
signatures = {
|
||
(row.pimcomp_schedule, row.pimcomp_validation,
|
||
*(None if math.isnan(value) else value for value in (
|
||
row.pimcomp_latency, row.pimcomp_throughput, row.pimcomp_power, row.pimcomp_energy)))
|
||
for row in throughput
|
||
}
|
||
if len(signatures) != 1:
|
||
raise ValueError(f"{arch}/{model}: inconsistent shared PIMCOMP batch results across pipelines")
|
||
columns.append((throughput[0], "pimcomp"))
|
||
return columns
|
||
|
||
|
||
def label_groups(ax, results: list[Result], positions: list[float] | None = None) -> None:
|
||
if positions is None:
|
||
positions = list(range(len(results)))
|
||
centers, labels = [], []
|
||
start = 0
|
||
for index, (model, rows) in enumerate(groupby(results, key=lambda row: row.model)):
|
||
end = start + len(list(rows))
|
||
centers.append((positions[start] + positions[end - 1]) / 2)
|
||
labels.append(MODEL_NAMES[model])
|
||
left = ax.get_xlim()[0] if start == 0 else (positions[start - 1] + positions[start]) / 2
|
||
right = ax.get_xlim()[1] if end == len(results) else (positions[end - 1] + positions[end]) / 2
|
||
if index % 2 == 0:
|
||
ax.axvspan(left, right, facecolor=BAND_COLOR, zorder=0)
|
||
if index > 0:
|
||
ax.axvline(left, color=LINE_COLOR, linewidth=0.7)
|
||
start = end
|
||
ax.set_xticks(centers, labels, fontsize=9.5)
|
||
|
||
|
||
def grouped_positions(results: list[Result]) -> list[float]:
|
||
positions = []
|
||
for index, row in enumerate(results):
|
||
if index == 0:
|
||
positions.append(0.0)
|
||
continue
|
||
previous = results[index - 1]
|
||
gap = 0.8 if row.model != previous.model else 0.4 if row.mode != previous.mode else 0.0
|
||
positions.append(positions[-1] + 1.0 + gap)
|
||
return positions
|
||
|
||
|
||
def diverging_ratio(ratio: float) -> float:
|
||
return ratio - 1 if ratio >= 1 else 1 - 1 / ratio
|
||
|
||
|
||
def draw_validation(results: list[Result], output_dir: Path, compact: bool = False) -> None:
|
||
cells = {}
|
||
for row, compiler in absolute_columns(results):
|
||
stage = row.pipeline if compiler == "raptor" else row.pimcomp_schedule
|
||
key = (row.arch, row.model, row.mode, None if compact else stage, compiler)
|
||
cells.setdefault(key, set()).add(getattr(row, f"{compiler}_validation"))
|
||
configurations = list(dict.fromkeys(key[2:] for key in cells))
|
||
positions = []
|
||
for index, (mode, _, _) in enumerate(configurations):
|
||
gap = 0.4 if index and mode != configurations[index - 1][0] else 0
|
||
positions.append(positions[-1] + 1 + gap if index else 0)
|
||
configuration_x = dict(zip(configurations, positions))
|
||
headers = [f"Raptor\n{stage} stages" if mode == "throughput" and compiler == "raptor" and not compact
|
||
else "PIMCOMP\nbatch" if mode == "throughput" and compiler == "pimcomp" and not compact
|
||
else COMPILER_NAMES[compiler]
|
||
for mode, stage, compiler in configurations]
|
||
fig, axes = plt.subplots(1, 2, figsize=(8.5 if compact else 12, 3.3), layout="constrained")
|
||
for ax, arch in zip(axes, ARCHITECTURES):
|
||
ax.set_xlim(positions[0] - 0.55, positions[-1] + 0.55)
|
||
ax.set_ylim(len(MODELS) - 0.5, -0.5)
|
||
centers, modes = [], []
|
||
for mode, group in groupby(zip(configurations, positions), key=lambda item: item[0][0]):
|
||
group_positions = [x for _, x in group]
|
||
centers.append((group_positions[0] + group_positions[-1]) / 2)
|
||
modes.append("Latency" if mode == "latency" else MODE_NAMES[mode])
|
||
ax.set_xticks(centers, modes, fontsize=10)
|
||
ax.set_xticks(positions, headers, minor=True)
|
||
ax.tick_params(axis="x", which="minor", top=True, labeltop=True,
|
||
bottom=False, labelbottom=False, length=0, pad=6, labelsize=9)
|
||
ax.set_yticks(range(len(MODELS)), [MODEL_NAMES[model] for model in MODELS], fontsize=9)
|
||
ax.tick_params(axis="both", length=0, pad=6)
|
||
ax.tick_params(axis="y", labelleft=arch == ARCHITECTURES[0])
|
||
architecture = ARCH_NAMES[arch]
|
||
ax.set_xlabel(architecture, fontsize=11, weight="bold", labelpad=8)
|
||
for (cell_arch, model, mode, stage, compiler), statuses in cells.items():
|
||
if cell_arch != arch:
|
||
continue
|
||
x, y = configuration_x[(mode, stage, compiler)], MODELS.index(model)
|
||
status = next(status for status in ("FAIL", "SKIPPED", "NA", "PASS") if status in statuses)
|
||
color, border = VALIDATION_STYLES[status]
|
||
ax.add_patch(Rectangle((x - 0.5, y - 0.42), 1, 0.84,
|
||
facecolor=color, edgecolor=border, linewidth=0.7))
|
||
ax.text(x, y, "N/A" if status == "NA" else status,
|
||
color=MATRIX_TEXT_COLOR, ha="center", va="center", fontsize=12, weight="normal")
|
||
fit_x_tick_labels(fig)
|
||
save_figure(fig, "validation_status_compact" if compact else "validation_status", output_dir)
|
||
|
||
|
||
def select_compact_speedups(results: list[Result]) -> list[Result]:
|
||
selected = []
|
||
for _, group in groupby(results, key=lambda row: (row.arch, row.model, row.mode)):
|
||
rows = list(group)
|
||
selected.append(max(rows, key=lambda row: row.speedup if math.isfinite(row.speedup) else -math.inf))
|
||
return selected
|
||
|
||
|
||
def draw_metric(results: list[Result], field: str, ylabel: str,
|
||
basename: str, output_dir: Path, diverging: bool = False,
|
||
compact: bool = False) -> None:
|
||
data = [diverging_ratio(getattr(row, field)) if diverging else getattr(row, field)
|
||
for row in results]
|
||
available = [v for v in data if math.isfinite(v)]
|
||
upper = max(0, max(available, default=1.0))
|
||
lower = min(0, min(available, default=0)) if diverging else 0
|
||
padding = max(upper - lower, 1.0) * 0.3
|
||
y_max = math.ceil((upper + padding) * 2) / 2
|
||
y_min = math.floor((lower - padding) * 2) / 2 if diverging else 0
|
||
y_interval = 1.0 if y_max - y_min > 4.5 else 0.5
|
||
|
||
fig, axes = plt.subplots(1, 2, sharey=True, figsize=(8.5 if compact else 12, 4.4),
|
||
layout="constrained")
|
||
if compact:
|
||
single_metric = "latency" if field == "speedup" else field
|
||
throughput_suffix = "" if field == "speedup" else f" {field}"
|
||
handles = [Patch(facecolor=METRIC_STYLES["latency"][0],
|
||
edgecolor=METRIC_STYLES["latency"][1],
|
||
label=f"Single-inference {single_metric}"),
|
||
Patch(facecolor=METRIC_STYLES["throughput"][0],
|
||
edgecolor=METRIC_STYLES["throughput"][1], hatch="//",
|
||
label=f"Best-case throughput{throughput_suffix}")]
|
||
|
||
for ax, arch in zip(axes, ARCHITECTURES):
|
||
selected = [row for row in results if row.arch == arch]
|
||
positions = [index + index // 2 for index in range(len(selected))] if compact else grouped_positions(selected)
|
||
ax.set_xlabel(ARCH_NAMES[arch],
|
||
fontsize=11, weight="bold", labelpad=8)
|
||
ax.set_axisbelow(True)
|
||
ax.set_ylim(y_min, y_max)
|
||
ax.set_xlim(-0.9, positions[-1] + 0.9 if positions else 0.9)
|
||
label_groups(ax, selected, positions)
|
||
ax.tick_params(axis="y", labelleft=arch == ARCHITECTURES[0], pad=5, length=0)
|
||
ax.tick_params(axis="x", labelsize=9.5, length=0, pad=6)
|
||
ax.yaxis.set_major_locator(MultipleLocator(y_interval))
|
||
if diverging:
|
||
ax.yaxis.set_major_formatter(FuncFormatter(lambda value, _: f"{abs(value) + 1:g}"))
|
||
ax.grid(axis="y")
|
||
ax.axhline(0 if diverging else 1, color=TEXT_COLOR, linewidth=1.0,
|
||
linestyle="--", zorder=3)
|
||
if arch == ARCHITECTURES[0]:
|
||
ax.set_ylabel(ylabel, fontsize=10.5, labelpad=6)
|
||
|
||
for position, row in zip(positions, selected):
|
||
ratio = getattr(row, field)
|
||
if not math.isfinite(ratio):
|
||
ax.text(position, (y_max - y_min) * 0.02, "N/A", ha="center", va="bottom",
|
||
fontsize=9.5, color=TEXT_COLOR, fontstyle="italic")
|
||
continue
|
||
compiler = advantage_compiler(ratio)
|
||
statuses = [status for status in (row.raptor_validation, row.pimcomp_validation) if status != "PASS"]
|
||
status = "FAIL" if "FAIL" in statuses else next(iter(statuses), "PASS")
|
||
pattern = STAGE_PATTERNS[row.pipeline]
|
||
color, border = COMPILER_STYLES[compiler] if compiler else ("white", TEXT_COLOR)
|
||
if compact:
|
||
pattern = "//" if row.mode == "throughput" else ""
|
||
color, border = METRIC_STYLES[row.mode]
|
||
measurement = diverging_ratio(ratio) if diverging else ratio
|
||
ax.bar(position, measurement, width=1.0,
|
||
color=color,
|
||
hatch=pattern,
|
||
edgecolor=border, linewidth=0.7, zorder=2)
|
||
direction = -1 if measurement < 0 else 1
|
||
factor = max(ratio, 1 / ratio) if diverging else ratio
|
||
value_label(ax, position, measurement + direction * (y_max - y_min) * 0.012,
|
||
f"{factor:.2f}×", status, direction, fontsize=9.1, invalid_label="P. INVALID")
|
||
|
||
fit_bar_widths(fig)
|
||
if compact:
|
||
add_legend(fig, handles)
|
||
else:
|
||
figure_legend(fig, results, field)
|
||
fit_x_tick_labels(fig)
|
||
fit_value_labels(fig)
|
||
save_figure(fig, basename, output_dir)
|
||
|
||
|
||
def draw_absolute(results: list[Result], field: str, ylabel: str,
|
||
basename: str, output_dir: Path, scale: float = 1.0,
|
||
raptor_only: bool = False) -> None:
|
||
columns = absolute_columns(results, raptor_only)
|
||
available = [getattr(row, f"{compiler}_{field}") * scale
|
||
for row, compiler in columns
|
||
if math.isfinite(getattr(row, f"{compiler}_{field}"))]
|
||
y_max = max(available, default=1.0) * 1.35
|
||
fig, axes = plt.subplots(1, 2, sharey=True, figsize=(12, 4.4),
|
||
layout="constrained")
|
||
|
||
for ax, arch in zip(axes, ARCHITECTURES):
|
||
selected = [(row, compiler) for row, compiler in columns if row.arch == arch]
|
||
positions = grouped_positions([row for row, _ in selected])
|
||
ax.set_xlabel(ARCH_NAMES[arch],
|
||
fontsize=11, weight="bold", labelpad=8)
|
||
ax.set_axisbelow(True)
|
||
ax.set_ylim(0, y_max)
|
||
if positions:
|
||
ax.set_xlim(positions[0] - 0.9, positions[-1] + 0.9)
|
||
else:
|
||
ax.set_xlim(-0.9, 0.9)
|
||
label_groups(ax, [row for row, _ in selected], positions)
|
||
if {row.mode for row, _ in selected} == {"latency", "throughput"}:
|
||
centers, labels = [], []
|
||
for (_, mode), group in groupby(enumerate(selected), key=lambda item: (item[1][0].model, item[1][0].mode)):
|
||
group_positions = [positions[position] for position, _ in group]
|
||
centers.append((group_positions[0] + group_positions[-1]) / 2)
|
||
labels.append("Latency" if mode == "latency" else MODE_NAMES[mode])
|
||
ax.set_xticks(centers, labels, minor=True)
|
||
ax.tick_params(axis="x", which="minor", length=0, pad=4, labelsize=8)
|
||
if arch == ARCHITECTURES[0]:
|
||
ax.set_ylabel(ylabel, fontsize=10.5)
|
||
if field == "throughput":
|
||
ax.ticklabel_format(axis="y", style="plain", useOffset=False)
|
||
ax.grid(axis="y")
|
||
ax.tick_params(axis="y", labelleft=arch == ARCHITECTURES[0], length=0, pad=6)
|
||
ax.tick_params(axis="x", which="major", length=0,
|
||
pad=22 if {row.mode for row, _ in selected} == {"latency", "throughput"} else 6)
|
||
|
||
for x, (row, compiler) in zip(positions, selected):
|
||
measurement = getattr(row, f"{compiler}_{field}") * scale
|
||
if not math.isfinite(measurement):
|
||
ax.text(x, y_max * 0.02, "N/A", rotation=90, ha="center",
|
||
va="bottom", fontsize=8, color=TEXT_COLOR)
|
||
continue
|
||
status = getattr(row, f"{compiler}_validation")
|
||
pattern = STAGE_PATTERNS[row.pipeline] if compiler == "raptor" else ""
|
||
fill, border = COMPILER_STYLES[compiler]
|
||
ax.bar(x, measurement, width=1.0,
|
||
color=fill,
|
||
hatch=pattern,
|
||
edgecolor=border, linewidth=0.7, zorder=2)
|
||
label = (f"{measurement:,.2f}".rstrip("0").rstrip(".")
|
||
if field == "throughput" else f"{measurement:.3g}")
|
||
value_label(ax, x, measurement + y_max * 0.012, label, status, invalid_label="P. INVALID")
|
||
|
||
fit_bar_widths(fig)
|
||
figure_legend(fig, results, field, absolute=True, raptor_only=raptor_only)
|
||
fit_x_tick_labels(fig)
|
||
fit_value_labels(fig)
|
||
save_figure(fig, basename, output_dir)
|
||
|
||
|
||
def draw_pipeline_effect(results: list[Result], output_dir: Path) -> None:
|
||
rows = [row for row in results if row.mode == "throughput" or row.pipeline == 1]
|
||
stages = sorted({row.pipeline for row in rows})
|
||
panels = [model for model in MODELS if any(row.model == model for row in rows)]
|
||
lookup = {(row.arch, row.model, row.pipeline): row for row in rows}
|
||
rates = {key: quotient(1000, row.raptor_latency) if row.mode == "latency" else row.raptor_throughput
|
||
for key, row in lookup.items()}
|
||
fig, axes = plt.subplots(len(ARCHITECTURES), len(panels), sharex=True, sharey="col",
|
||
figsize=(2.6 * len(panels), 4.3), squeeze=False, layout="constrained")
|
||
fig.supylabel("Throughput (inferences/s)", fontsize=10.5)
|
||
for column, model in enumerate(panels):
|
||
fill, border = NETWORK_STYLES[model]
|
||
for index, arch in enumerate(ARCHITECTURES):
|
||
ax = axes[index, column]
|
||
keys = [(arch, model, stage) for stage in stages]
|
||
values = [rates.get(key, math.nan) for key in keys]
|
||
valid = [key in lookup and lookup[key].raptor_validation == "PASS" for key in keys]
|
||
ax.plot(stages, values, color=border, linewidth=1.8)
|
||
for stage, value, passed in zip(stages, values, valid):
|
||
if math.isfinite(value):
|
||
marker = ("s" if stage == 1 else "o") if passed else "x"
|
||
status = lookup[(arch, model, stage)].raptor_validation
|
||
ax.plot(stage, value, color=ERROR_COLOR if status == "FAIL" else border,
|
||
markerfacecolor=fill, marker=marker,
|
||
markersize=6, linestyle="none")
|
||
ax.set_xticks(stages)
|
||
if index == 0:
|
||
ax.set_title(MODEL_NAMES[model], fontsize=10.5, weight="normal", pad=8)
|
||
ax.grid(axis="y")
|
||
ax.tick_params(axis="both", length=0, pad=6,
|
||
labelbottom=index == len(ARCHITECTURES) - 1, labelleft=True)
|
||
ax.ticklabel_format(axis="y", style="plain", useOffset=False)
|
||
if column == len(panels) - 1:
|
||
ax.yaxis.set_label_position("right")
|
||
ax.set_ylabel(ARCH_NAMES[arch], fontsize=10.5, weight="bold", rotation=270, labelpad=24)
|
||
for ax in axes.flat:
|
||
ax.margins(y=0.08)
|
||
handles = [Line2D([], [], color=TEXT_COLOR, markerfacecolor="white",
|
||
marker=marker, linestyle="none", label=label)
|
||
for marker, label in (("s", "Derived as 1 / latency"),
|
||
("o", "Throughput measurement"))]
|
||
if any(row.raptor_validation != "PASS" and math.isfinite(rates[key]) for key, row in lookup.items()):
|
||
handles.append(Line2D([], [], color=TEXT_COLOR, marker="x", linestyle="none",
|
||
label="Raptor validation not PASS"))
|
||
add_legend(fig, handles)
|
||
fig.supxlabel("Raptor pipeline stages", fontsize=11)
|
||
fit_x_tick_labels(fig)
|
||
save_figure(fig, "raptor_pipeline_throughput", output_dir)
|
||
|
||
|
||
def main() -> None:
|
||
parser = argparse.ArgumentParser(description=__doc__)
|
||
script_dir = Path(__file__).resolve().parent
|
||
parser.add_argument("--input", type=Path, default=script_dir / "results_comparison.csv")
|
||
parser.add_argument("--output-dir", type=Path, default=script_dir / "plots")
|
||
args = parser.parse_args()
|
||
configure_style()
|
||
results = load_results(args.input)
|
||
valid = sum(row.comparison_passed for row in results)
|
||
print(f"Results: {len(results)} configurations, {valid} valid comparisons, "
|
||
f"{len(results)-valid} comparisons without two PASS statuses")
|
||
draw_validation(results, args.output_dir)
|
||
draw_validation(results, args.output_dir, compact=True)
|
||
draw_pipeline_effect(results, args.output_dir)
|
||
yolo_only_models = {
|
||
model for model in MODELS if model.startswith("yolo")
|
||
and not any(math.isfinite(getattr(row, f"pimcomp_{field}"))
|
||
for row in results if row.model == model
|
||
for field in ("latency", "power", "energy"))
|
||
}
|
||
comparisons = [row for row in results if row.model not in yolo_only_models]
|
||
yolo_only = [row for row in results if row.model in yolo_only_models]
|
||
compact_speedups = select_compact_speedups(comparisons)
|
||
for metric, label in (("speedup", "Speedup (x)"),
|
||
("energy", "Energy-per-inference advantage (x)"),
|
||
("power", "Average power advantage (x)")):
|
||
for suffix, diverging in (("comparison", False), ("diverging", True)):
|
||
draw_metric(compact_speedups, metric, label,
|
||
f"{metric}_compact_{suffix}", args.output_dir, diverging=diverging, compact=True)
|
||
for metric, name, label, mode in [
|
||
("latency_speedup", "longest_stage_latency", "Longest stage latency speedup (x)", None),
|
||
("latency_speedup", "latency", "Single-inference latency speedup (x)", "latency"),
|
||
("throughput_speedup", "throughput", "Throughput speedup (x)", "throughput"),
|
||
("power", "power", "Average power advantage (x)", None),
|
||
("energy", "energy", "Energy-per-inference advantage (x)", None),
|
||
]:
|
||
selected = [row for row in comparisons if mode is None or row.mode == mode]
|
||
draw_metric(selected, metric, label,
|
||
f"{name}_comparison", args.output_dir)
|
||
draw_metric(selected, metric, label,
|
||
f"{name}_diverging", args.output_dir, diverging=True)
|
||
for metric, name, label, scale, mode in [
|
||
("latency", "longest_stage_latency", "Longest stage latency (ms)", 1.0, None),
|
||
("latency", "latency", "Single-inference latency (ms)", 1.0, "latency"),
|
||
("power", "power", "Average power (mW)", 1.0, None),
|
||
("energy", "energy", "Energy per inference (mJ)", 1e-9, None),
|
||
("throughput", "throughput", "Throughput (inferences/s)", 1.0, "throughput"),
|
||
]:
|
||
for suffix, group, raptor_only in (
|
||
("absolute", comparisons, False), ("yolo", yolo_only, True)
|
||
):
|
||
selected = [row for row in group if mode is None or row.mode == mode]
|
||
if not selected:
|
||
continue
|
||
draw_absolute(selected, metric, label,
|
||
f"{name}_{suffix}", args.output_dir, scale,
|
||
raptor_only=raptor_only)
|
||
|
||
|
||
if __name__ == "__main__":
|
||
main()
|