update plots
Validate Operations / validate-operations (push) Has been cancelled

This commit is contained in:
NiccoloN
2026-10-26 12:18:36 +01:00
parent 4b53f6cf10
commit 49d6fb42cf
8 changed files with 918 additions and 370 deletions
+4 -22
View File
@@ -47,27 +47,9 @@ guarantees, so those variants may fail numerical validation and exit nonzero.
Baseline and `no-plan` must pass. Percentages from failing variants are not
valid speedups; deadline averages do not prove steady-state convergence.
## Plots
## Result data
```bash
.venv/bin/python validation/tools/pim/ablation/generate_plots.py
# For a custom experiment root:
.venv/bin/python validation/tools/pim/ablation/generate_plots.py \
--input /path/to/experiment/results_ablation.csv
```
Requires Matplotlib from the root `requirements.txt`. By default, reads the
canonical CSV beside this script. Plots go to
`plots/png/` and `plots/svg/` beside the selected CSV; override with
`--output-dir PATH`.
Single-variant comparison CSVs are not accepted.
The checked-in CSV is an intentional result dataset; plotting uses its current
contents without running benchmarks. Invalid reciprocal percentage pairs render
as `N/A`; individual legitimate extreme changes are preserved. See
[performance availability](../../../../docs/evaluation/methodology.md#functional-validity-and-performance-availability).
Plots separate modes and pipelines. Throughput-mode latency means output
interval. Failed validation is labeled `INVALID`, other non-PASS statuses keep
their names, and missing measurements show `N/A`. See the
The checked-in `results_ablation.csv` is an intentional result dataset for tabular
reporting. Ablation experiments and aggregation remain available through the
runner above. Percentages from failing variants are not valid speedups. See the
[comparison metric definitions](../../../../docs/evaluation/methodology.md#metric-interpretation).
@@ -1,156 +0,0 @@
#!/usr/bin/env python3
"""Plot percentage changes from an experiment's combined results_ablation.csv."""
from __future__ import annotations
import argparse
import csv
import math
import sys
import warnings
from dataclasses import dataclass
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from plot_style import COLORS, configure_style, save_figure, value_label
import matplotlib.pyplot as plt
from matplotlib.patches import Patch
from matplotlib.ticker import PercentFormatter
from run_ablation import ABLATION_DEFAULT_MODELS, DEFAULT_RESULTS_PATH, RESULT_FIELDS, VARIANTS
VARIANT_COLORS = {variant.name: color for variant, color in zip(VARIANTS, COLORS)}
STATUSES = ("PASS", "FAIL", "SKIPPED", "NA")
@dataclass(frozen=True)
class Result:
variant: str
arch: str
model: str
mode: str
pipeline: int
validation: str
metrics: dict[str, float]
def percentage(value: str) -> float:
if value == "NA":
return math.nan
if not value.endswith("%"):
raise ValueError(f"Expected a percentage or NA, got {value!r}")
parsed = float(value[:-1])
if not math.isfinite(parsed):
raise ValueError(f"Non-finite percentage: {value!r}")
return parsed
def load_results(path: Path) -> list[Result]:
rows = []
seen = set()
invalid_pairs = []
with path.open(newline="", encoding="utf-8-sig") as stream:
reader = csv.DictReader(stream)
if not {"variant", *RESULT_FIELDS} <= set(reader.fieldnames or []):
raise ValueError("Expected the combined ablation CSV with variant and percentage columns; "
"a per-variant results_comparison.csv cannot be used here.")
for row in reader:
variant, mode, status = row["variant"], row["mode"], row["functional_validation"]
if variant not in VARIANT_COLORS or mode not in ("latency", "throughput") or status not in STATUSES:
raise ValueError(f"Unknown ablation variant, mode, or validation status: {variant}, {mode}, {status}")
pipeline = int(row["raptor_pipeline"])
key = (variant, row["arch"], row["model"], mode, pipeline)
if key in seen:
raise ValueError(f"Duplicate ablation configuration: {key}")
seen.add(key)
metrics = {metric: percentage(row[metric])
for metric in ("latency", "throughput", "power", "energy")}
# Each may round to -100% alone; reciprocal metrics cannot both do so.
if mode == "throughput" and metrics["latency"] == metrics["throughput"] == -100:
metrics["latency"] = metrics["throughput"] = math.nan
invalid_pairs.append(key)
rows.append(Result(variant, row["arch"], row["model"], mode, pipeline, status, metrics))
if not rows:
raise ValueError("The ablation CSV contains no results")
if invalid_pairs:
warnings.warn(f"{len(invalid_pairs)} invalid reciprocal latency/throughput percentage pairs; "
f"displaying N/A. Examples: {invalid_pairs[:3]}", stacklevel=2)
return rows
def draw_metric(results: list[Result], metric: str, ylabel: str, output_dir: Path) -> None:
architectures = sorted({row.arch for row in results})
present_models = {row.model for row in results}
models = [model for model in ABLATION_DEFAULT_MODELS if model in present_models]
models += sorted(present_models - set(models))
variants = [name for name in VARIANT_COLORS if any(row.variant == name for row in results)]
lookup = {(row.arch, row.model, row.variant): row for row in results}
values = [row.metrics[metric] for row in results if math.isfinite(row.metrics[metric])]
lower, upper = min(0, min(values, default=0)), max(0, max(values, default=0))
padding = max(upper - lower, 1) * 0.35
fig, axes = plt.subplots(1, len(architectures), sharey=True, squeeze=False,
figsize=(7 * len(architectures), 5.2), layout="constrained")
handles = [Patch(facecolor=VARIANT_COLORS[name], label=name) for name in variants]
if len(values) < len(architectures) * len(models) * len(variants):
handles.append(Patch(facecolor="white", edgecolor="#B2BAC5", label="N/A: missing measurement"))
fig.legend(handles=handles, loc="outside upper center", ncol=4, frameon=False, fontsize=9.5)
width = 0.8 / len(variants)
mode, pipeline = results[0].mode, results[0].pipeline
for ax, arch in zip(axes[0], architectures):
ax.set_axisbelow(True)
ax.set_ylim(lower - padding if lower < 0 else 0, upper + padding)
ax.set_xlim(-0.5, len(models) - 0.5)
ax.set_xticks(range(len(models)), [model.upper() for model in models], fontsize=9.5)
ax.set_xlabel(f"{arch.replace('arch-', 'ARCHITECTURE ').upper()}\n"
f"{mode.capitalize()}, {pipeline} stage" + ("s" if pipeline != 1 else ""),
fontsize=11, weight="bold", labelpad=8)
ax.set_ylabel(ylabel, fontsize=10.5)
ax.yaxis.set_major_formatter(PercentFormatter(xmax=100, decimals=0))
ax.tick_params(axis="both", length=0, pad=6, labelleft=True)
ax.grid(axis="y", color="#E6EAF0", linewidth=0.8)
ax.axhline(0, color="#536478", linewidth=1.2, linestyle="--")
for index, model in enumerate(models):
if index % 2 == 0:
ax.axvspan(index - 0.5, index + 0.5, facecolor="#F7F9FC", zorder=0)
for offset, variant in enumerate(variants):
x = index + (offset - (len(variants) - 1) / 2) * width
row = lookup.get((arch, model, variant))
value = row.metrics[metric] if row else math.nan
if not math.isfinite(value):
ax.text(x, padding * 0.05, "N/A", rotation=90, ha="center", va="bottom",
fontsize=8, color="#85909F")
continue
ax.bar(x, value, width=width * 0.9, color=VARIANT_COLORS[variant],
edgecolor="none", linewidth=0, zorder=2)
direction = -1 if value < 0 else 1
value_label(ax, x, value + direction * padding * 0.05, f"{value:+,.2f}%",
row.validation, direction)
save_figure(fig, f"{mode}_pipeline{pipeline}_{metric}", output_dir)
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--input", type=Path, default=DEFAULT_RESULTS_PATH)
parser.add_argument("--output-dir", type=Path)
args = parser.parse_args()
if args.output_dir is None:
args.output_dir = args.input.parent / "plots"
configure_style()
results = load_results(args.input)
print(f"Results: {len(results)} configurations, "
f"{sum(row.validation != 'PASS' for row in results)} without PASS validation")
for mode, pipeline in sorted({(row.mode, row.pipeline) for row in results}):
rows = [row for row in results if row.mode == mode and row.pipeline == pipeline]
for metric, label in (
("latency", "Single-inference latency" if mode == "latency" else "Output interval"),
("throughput", "Throughput"), ("power", "Average power"), ("energy", "Energy per inference"),
):
if metric == "throughput" and mode == "latency":
continue
draw_metric(rows, metric, f"{label} change from baseline (%)", args.output_dir)
if __name__ == "__main__":
main()
+26 -2
View File
@@ -72,7 +72,7 @@ The comparator defaults to absolute/relative tolerances `1e-3`/`1e-4`;
Requires Matplotlib from the root `requirements.txt`. Defaults to the supplied
`results_comparison.csv` beside the script and writes `plots/png/` and
`plots/svg/` there. Use `--input PATH` for a runner-generated CSV and
`plots/svg/` and `plots/pdf/` there. Use `--input PATH` for a runner-generated CSV and
`--output-dir PATH` to override the destination. The plotter requires the full
40-case matrix: five models × Arch-A/B × latency pipeline 1 and throughput
pipelines 2/4/8. Duplicate, missing, unexpected, or inconsistent shared Pimcomp
@@ -81,6 +81,9 @@ configurations are rejected before plots are written.
| Plot family | Meaning |
|---|---|
| `validation_status` | Recorded functional status for each compiler and experiment. |
| `validation_status_compact` | Compact functional status matrix. |
| `speedup_compact_*` | Touching single-inference latency and best-pipeline throughput bars for each network. |
| `energy_compact_*`, `power_compact_*` | Energy and power advantages for the same cases selected by the compact speedup plots. |
| `latency_*` | Single-inference latency, latency mode only. |
| `throughput_*` | Inferences/s, throughput mode only. |
| `longest_stage_latency_*` | Combined single-inference latency and throughput output interval; the latter is a stage-latency proxy. |
@@ -93,7 +96,28 @@ advantages in either direction. Ratios above 1 favor Raptor: Pimcomp/Raptor for
latency, power, and energy; Raptor/Pimcomp for throughput. Diverging bars use
winning factor minus one, with parity at zero.
Failed validation is labeled `INVALID`; other non-PASS statuses keep their
The shared `plot_style.py` defines the manuscript palette and display names.
Raptor uses sage green; PIMCOMP uses plain sand. Stage patterns,
outlines, marker shapes, and status text preserve distinctions in grayscale.
Compact plots use blue for the single-inference case and hatched sage
green for the throughput case.
Each architecture/network pair uses the highest recorded throughput across pipelines.
Energy and power use those same cases, without selecting a different pipeline for each metric.
Validation labels remain visible for the selected cases. Units appear in axis
titles and bar values; axis tick labels contain numbers only.
Throughput-stage bars touch within each network in both absolute and advantage
plots. Bar-label widths are measured before export; dense figures expand only
as needed to contain values and invalidity annotations. Legends use the available
width and balance entries across rows when wrapping is necessary.
Comparison panels share y-axis titles and tick labels. Each network in the
pipeline plot retains its own scale, with y-axis ticks on both architecture rows.
Only the bottom row shows the shared x-axis ticks. Network names appear above
the top row; architecture names appear on the right. That plot
trims unused vertical range and is intended to span both manuscript columns.
Failed validation is labeled `P. INVALID`; other non-PASS statuses keep their
names, and missing measurements show `N/A`. Relative bars require both
compilers to pass to be valid. Repeated Pimcomp batch measurements and statuses
must agree before being collapsed. `FAIL` alone does not identify the cause.
@@ -1,25 +1,25 @@
#!/usr/bin/env python3
"""Generate publication-ready Raptor vs Pimcomp comparison charts.
"""Generate publication-ready Raptor vs PIMCOMP comparison charts.
Usage:
.venv/bin/python validation/tools/pim/pimcomp/compare/generate_plots.py
Dependencies: matplotlib (no pandas or seaborn required).
By default, reads results_comparison.csv beside this script and writes plots/
beside it, with outputs grouped in png/ and svg/ under the output directory.
beside it, with outputs grouped in png/, svg/, and pdf/ under the output directory.
YOLO models without Pimcomp measurements get separate Raptor-only charts.
YOLO models without PIMCOMP measurements get separate Raptor-only charts.
Comparisons use vertical bars for ratios, diverging advantages, and absolute
values, with compiler colors and patterns for the Raptor pipeline stages.
Throughput-mode latency is the output interval, a bottleneck-stage proxy,
not end-to-end inference latency. Separate throughput plots use inferences/s.
Energy is reported per inference; power is averaged over the simulation.
Numerical results whose compiler did not pass functional validation have a red
status following their value label; missing Pimcomp
status following their value label; missing PIMCOMP
measurements appear as 'N/A'. The fixed compiler palette is shared by
all metrics. Relative plots use Pimcomp / Raptor for latency, power and energy,
and Raptor / Pimcomp for throughput, so >1 favors Raptor. Diverging plots put the winner's
factor above (Raptor) or below (Pimcomp) parity, with 1x at the zero line.
all metrics. Relative plots use PIMCOMP / Raptor for latency, power and energy,
and Raptor / PIMCOMP for throughput, so >1 favors Raptor. Diverging plots put the winner's
factor above (Raptor) or below (PIMCOMP) parity, with 1x at the zero line.
"""
from __future__ import annotations
@@ -39,20 +39,22 @@ from matplotlib.lines import Line2D
from matplotlib.ticker import FuncFormatter, MultipleLocator
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
from plot_style import COLORS, HATCH_COLOR, TEXT_COLOR, configure_style, fit_value_labels, save_figure, value_label
from plot_style import (ARCH_NAMES, BAND_COLOR, COMPILER_NAMES, COMPILER_STYLES,
ERROR_COLOR, LINE_COLOR, MATRIX_TEXT_COLOR, METRIC_STYLES,
MODE_NAMES, MODEL_NAMES, NETWORK_STYLES,
PALETTE, TEXT_COLOR, add_legend, configure_style, fit_value_labels,
fit_bar_widths, fit_x_tick_labels, save_figure, value_label)
COMPILER_COLORS = {"raptor": COLORS[0], "pimcomp": COLORS[2]}
STAGE_PATTERNS = {1: "", 2: "..", 4: "//", 8: "\\\\"}
ARCHITECTURES = ("arch-a", "arch-b")
MODELS = ("vgg8", "resnet18", "resnet34", "googlenet", "yolo11n")
CONFIGURATIONS = (("latency", 1), ("throughput", 2), ("throughput", 4), ("throughput", 8))
LINE_COLOR = "#A8B2C0"
VALIDATION_STYLES = {
"PASS": ("#168C8B", "white", ""),
"FAIL": ("#D18D32", HATCH_COLOR, ""),
"SKIPPED": ("#E7DFF0", TEXT_COLOR, ""),
"NA": ("#E8EDF1", TEXT_COLOR, ""),
"PASS": PALETTE["sage"],
"FAIL": PALETTE["red"],
"SKIPPED": PALETTE["pale_sand"],
"NA": PALETTE["pale_blue"],
}
@@ -85,6 +87,10 @@ class Result:
def throughput_speedup(self) -> float:
return quotient(self.raptor_throughput, self.pimcomp_throughput)
@property
def speedup(self) -> float:
return self.latency_speedup if self.mode == "latency" else self.throughput_speedup
def positive_float(text: str | None) -> float:
try:
@@ -129,7 +135,7 @@ def load_results(path: Path) -> list[Result]:
seen[key] = line
schedule = row["pimcomp_pipeline"].strip().lower()
if schedule != ("element" if mode == "latency" else "batch"):
raise ValueError(f"{key}: incompatible Pimcomp schedule {schedule!r}")
raise ValueError(f"{key}: incompatible PIMCOMP schedule {schedule!r}")
rows.append(Result(
arch=arch,
model=model,
@@ -185,20 +191,20 @@ def figure_legend(fig, results: list[Result], field: str,
for _, compiler, value in available} - {None}
if len(visible_compilers) > 1:
for compiler in compilers:
handles.append(Patch(facecolor=COMPILER_COLORS[compiler],
label=compiler.capitalize() + ("" if absolute else " advantage")))
fill, border = COMPILER_STYLES[compiler]
handles.append(Patch(facecolor=fill, edgecolor=border,
label=COMPILER_NAMES[compiler] + ("" if absolute else " advantage")))
handles += [Patch(facecolor="white", edgecolor=TEXT_COLOR, hatch=STAGE_PATTERNS[depth],
label=f"{depth} stage" + (" (latency)" if depth == 1 else "s"))
label=("1 stage (latency)" if absolute else "Single-inference")
if depth == 1 else f"{depth} stages")
for depth in sorted(stages) if stages != {1}]
styles = []
if len(available) < len(measurements):
styles.append(Patch(facecolor="white", edgecolor="#B2BAC5",
styles.append(Patch(facecolor="white", edgecolor=LINE_COLOR,
label="N/A: missing measurement"))
handles += styles
if handles:
fig.legend(handles=handles, loc="outside upper center", ncol=(len(handles) + 1) // 2,
frameon=False, fontsize=9.5, handlelength=1.7, columnspacing=1.5,
handletextpad=0.6, labelspacing=0.4)
add_legend(fig, handles)
def absolute_columns(results: list[Result], raptor_only: bool = False) -> list[tuple[Result, str]]:
@@ -218,7 +224,7 @@ def absolute_columns(results: list[Result], raptor_only: bool = False) -> list[t
for row in throughput
}
if len(signatures) != 1:
raise ValueError(f"{arch}/{model}: inconsistent shared Pimcomp batch results across pipelines")
raise ValueError(f"{arch}/{model}: inconsistent shared PIMCOMP batch results across pipelines")
columns.append((throughput[0], "pimcomp"))
return columns
@@ -231,34 +237,50 @@ def label_groups(ax, results: list[Result], positions: list[float] | None = None
for index, (model, rows) in enumerate(groupby(results, key=lambda row: row.model)):
end = start + len(list(rows))
centers.append((positions[start] + positions[end - 1]) / 2)
labels.append(model.upper())
labels.append(MODEL_NAMES[model])
left = ax.get_xlim()[0] if start == 0 else (positions[start - 1] + positions[start]) / 2
right = ax.get_xlim()[1] if end == len(results) else (positions[end - 1] + positions[end]) / 2
if index % 2 == 0:
ax.axvspan(left, right, facecolor="#F7F9FC", zorder=0)
ax.axvspan(left, right, facecolor=BAND_COLOR, zorder=0)
if index > 0:
ax.axvline(left, color="#DDE3EB", linewidth=0.9)
ax.axvline(left, color=LINE_COLOR, linewidth=0.7)
start = end
ax.set_xticks(centers, labels, fontsize=9.5)
def grouped_positions(results: list[Result]) -> list[float]:
positions = []
for index, row in enumerate(results):
if index == 0:
positions.append(0.0)
continue
previous = results[index - 1]
gap = 0.8 if row.model != previous.model else 0.4 if row.mode != previous.mode else 0.0
positions.append(positions[-1] + 1.0 + gap)
return positions
def diverging_ratio(ratio: float) -> float:
return ratio - 1 if ratio >= 1 else 1 - 1 / ratio
def draw_validation(results: list[Result], output_dir: Path) -> None:
columns = absolute_columns(results)
configurations = list(dict.fromkeys((row.mode, row.pipeline if compiler == "raptor" else row.pimcomp_schedule,
compiler) for row, compiler in columns))
def draw_validation(results: list[Result], output_dir: Path, compact: bool = False) -> None:
cells = {}
for row, compiler in absolute_columns(results):
stage = row.pipeline if compiler == "raptor" else row.pimcomp_schedule
key = (row.arch, row.model, row.mode, None if compact else stage, compiler)
cells.setdefault(key, set()).add(getattr(row, f"{compiler}_validation"))
configurations = list(dict.fromkeys(key[2:] for key in cells))
positions = []
for index, (mode, _, _) in enumerate(configurations):
gap = 0.4 if index and mode != configurations[index - 1][0] else 0
positions.append(positions[-1] + 1 + gap if index else 0)
configuration_x = dict(zip(configurations, positions))
headers = [f"Raptor\n{stage} stages" if mode == "throughput" and compiler == "raptor"
else "Pimcomp\nbatch" if mode == "throughput" else compiler.capitalize()
headers = [f"Raptor\n{stage} stages" if mode == "throughput" and compiler == "raptor" and not compact
else "PIMCOMP\nbatch" if mode == "throughput" and compiler == "pimcomp" and not compact
else COMPILER_NAMES[compiler]
for mode, stage, compiler in configurations]
fig, axes = plt.subplots(1, 2, figsize=(14, 3.8), layout="constrained")
fig, axes = plt.subplots(1, 2, figsize=(8.5 if compact else 12, 3.3), layout="constrained")
for ax, arch in zip(axes, ARCHITECTURES):
ax.set_xlim(positions[0] - 0.55, positions[-1] + 0.55)
ax.set_ylim(len(MODELS) - 0.5, -0.5)
@@ -266,31 +288,41 @@ def draw_validation(results: list[Result], output_dir: Path) -> None:
for mode, group in groupby(zip(configurations, positions), key=lambda item: item[0][0]):
group_positions = [x for _, x in group]
centers.append((group_positions[0] + group_positions[-1]) / 2)
modes.append(mode.capitalize())
modes.append("Latency" if mode == "latency" else MODE_NAMES[mode])
ax.set_xticks(centers, modes, fontsize=10)
ax.set_xticks(positions, headers, minor=True)
ax.tick_params(axis="x", which="minor", top=True, labeltop=True,
bottom=False, labelbottom=False, length=0, pad=6, labelsize=9)
ax.set_yticks(range(len(MODELS)), [model.upper() for model in MODELS], fontsize=9)
ax.set_yticks(range(len(MODELS)), [MODEL_NAMES[model] for model in MODELS], fontsize=9)
ax.tick_params(axis="both", length=0, pad=6)
architecture = "ARCHITECTURE A" if arch == "arch-a" else "ARCHITECTURE B"
ax.tick_params(axis="y", labelleft=arch == ARCHITECTURES[0])
architecture = ARCH_NAMES[arch]
ax.set_xlabel(architecture, fontsize=11, weight="bold", labelpad=8)
for row, compiler in columns:
if row.arch != arch:
for (cell_arch, model, mode, stage, compiler), statuses in cells.items():
if cell_arch != arch:
continue
key = (row.mode, row.pipeline if compiler == "raptor" else row.pimcomp_schedule, compiler)
x, y = configuration_x[key], MODELS.index(row.model)
status = getattr(row, f"{compiler}_validation")
color, text_color, _ = VALIDATION_STYLES[status]
x, y = configuration_x[(mode, stage, compiler)], MODELS.index(model)
status = next(status for status in ("FAIL", "SKIPPED", "NA", "PASS") if status in statuses)
color, border = VALIDATION_STYLES[status]
ax.add_patch(Rectangle((x - 0.5, y - 0.42), 1, 0.84,
facecolor=color, edgecolor="#657081", linewidth=0.5))
facecolor=color, edgecolor=border, linewidth=0.7))
ax.text(x, y, "N/A" if status == "NA" else status,
color=text_color, ha="center", va="center", fontsize=9)
save_figure(fig, "validation_status", output_dir)
color=MATRIX_TEXT_COLOR, ha="center", va="center", fontsize=12, weight="normal")
fit_x_tick_labels(fig)
save_figure(fig, "validation_status_compact" if compact else "validation_status", output_dir)
def select_compact_speedups(results: list[Result]) -> list[Result]:
selected = []
for _, group in groupby(results, key=lambda row: (row.arch, row.model, row.mode)):
rows = list(group)
selected.append(max(rows, key=lambda row: row.speedup if math.isfinite(row.speedup) else -math.inf))
return selected
def draw_metric(results: list[Result], field: str, ylabel: str,
basename: str, output_dir: Path, diverging: bool = False) -> None:
basename: str, output_dir: Path, diverging: bool = False,
compact: bool = False) -> None:
data = [diverging_ratio(getattr(row, field)) if diverging else getattr(row, field)
for row in results]
available = [v for v in data if math.isfinite(v)]
@@ -301,49 +333,68 @@ def draw_metric(results: list[Result], field: str, ylabel: str,
y_min = math.floor((lower - padding) * 2) / 2 if diverging else 0
y_interval = 1.0 if y_max - y_min > 4.5 else 0.5
fig, axes = plt.subplots(1, 2, sharey=True, figsize=(14, 5.2),
fig, axes = plt.subplots(1, 2, sharey=True, figsize=(8.5 if compact else 12, 4.4),
layout="constrained")
figure_legend(fig, results, field)
if compact:
single_metric = "latency" if field == "speedup" else field
throughput_suffix = "" if field == "speedup" else f" {field}"
handles = [Patch(facecolor=METRIC_STYLES["latency"][0],
edgecolor=METRIC_STYLES["latency"][1],
label=f"Single-inference {single_metric}"),
Patch(facecolor=METRIC_STYLES["throughput"][0],
edgecolor=METRIC_STYLES["throughput"][1], hatch="//",
label=f"Best-case throughput{throughput_suffix}")]
for ax, arch in zip(axes, ARCHITECTURES):
selected = [row for row in results if row.arch == arch]
ax.set_xlabel("ARCHITECTURE A" if arch == "arch-a" else "ARCHITECTURE B",
positions = [index + index // 2 for index in range(len(selected))] if compact else grouped_positions(selected)
ax.set_xlabel(ARCH_NAMES[arch],
fontsize=11, weight="bold", labelpad=8)
ax.set_axisbelow(True)
ax.set_ylim(y_min, y_max)
ax.set_xlim(-0.5, len(selected) - 0.5)
label_groups(ax, selected)
ax.tick_params(axis="y", labelleft=True, pad=5, length=0)
ax.set_xlim(-0.9, positions[-1] + 0.9 if positions else 0.9)
label_groups(ax, selected, positions)
ax.tick_params(axis="y", labelleft=arch == ARCHITECTURES[0], pad=5, length=0)
ax.tick_params(axis="x", labelsize=9.5, length=0, pad=6)
ax.yaxis.set_major_locator(MultipleLocator(y_interval))
if diverging:
ax.yaxis.set_major_formatter(FuncFormatter(lambda value, _: f"{abs(value) + 1:g}×"))
ax.grid(axis="y", color="#E6EAF0", linewidth=0.8)
ax.axhline(0 if diverging else 1, color="#536478", linewidth=1.2,
ax.yaxis.set_major_formatter(FuncFormatter(lambda value, _: f"{abs(value) + 1:g}"))
ax.grid(axis="y")
ax.axhline(0 if diverging else 1, color=TEXT_COLOR, linewidth=1.0,
linestyle="--", zorder=3)
ax.set_ylabel(ylabel, fontsize=10.5, labelpad=6)
if arch == ARCHITECTURES[0]:
ax.set_ylabel(ylabel, fontsize=10.5, labelpad=6)
for position, row in enumerate(selected):
for position, row in zip(positions, selected):
ratio = getattr(row, field)
if not math.isfinite(ratio):
ax.text(position, (y_max - y_min) * 0.02, "N/A", ha="center", va="bottom",
fontsize=9.5, color="#85909F", fontstyle="italic")
fontsize=9.5, color=TEXT_COLOR, fontstyle="italic")
continue
compiler = advantage_compiler(ratio)
statuses = [status for status in (row.raptor_validation, row.pimcomp_validation) if status != "PASS"]
status = "FAIL" if "FAIL" in statuses else next(iter(statuses), "PASS")
pattern = STAGE_PATTERNS[row.pipeline]
color, border = COMPILER_STYLES[compiler] if compiler else ("white", TEXT_COLOR)
if compact:
pattern = "//" if row.mode == "throughput" else ""
color, border = METRIC_STYLES[row.mode]
measurement = diverging_ratio(ratio) if diverging else ratio
ax.bar(position, measurement, width=0.72,
color=COMPILER_COLORS.get(compiler, "#657081"),
ax.bar(position, measurement, width=1.0,
color=color,
hatch=pattern,
edgecolor=TEXT_COLOR if pattern else "none",
linewidth=0.7 if pattern else 0.0, zorder=2)
edgecolor=border, linewidth=0.7, zorder=2)
direction = -1 if measurement < 0 else 1
factor = max(ratio, 1 / ratio) if diverging else ratio
value_label(ax, position, measurement + direction * (y_max - y_min) * 0.012,
f"{factor:.2f}×", status, direction, fontsize=9.1)
f"{factor:.2f}×", status, direction, fontsize=9.1, invalid_label="P. INVALID")
fit_bar_widths(fig)
if compact:
add_legend(fig, handles)
else:
figure_legend(fig, results, field)
fit_x_tick_labels(fig)
fit_value_labels(fig)
save_figure(fig, basename, output_dir)
@@ -356,21 +407,13 @@ def draw_absolute(results: list[Result], field: str, ylabel: str,
for row, compiler in columns
if math.isfinite(getattr(row, f"{compiler}_{field}"))]
y_max = max(available, default=1.0) * 1.35
fig, axes = plt.subplots(1, 2, sharey=True, figsize=(14, 5.2),
fig, axes = plt.subplots(1, 2, sharey=True, figsize=(12, 4.4),
layout="constrained")
figure_legend(fig, results, field, absolute=True, raptor_only=raptor_only)
for ax, arch in zip(axes, ARCHITECTURES):
selected = [(row, compiler) for row, compiler in columns if row.arch == arch]
positions = []
for index, (row, _) in enumerate(selected):
if index == 0:
positions.append(0.0)
continue
previous = selected[index - 1][0]
gap = 0.8 if row.model != previous.model else 0.4 if row.mode != previous.mode else 0.0
positions.append(positions[-1] + 1.0 + gap)
ax.set_xlabel("ARCHITECTURE A" if arch == "arch-a" else "ARCHITECTURE B",
positions = grouped_positions([row for row, _ in selected])
ax.set_xlabel(ARCH_NAMES[arch],
fontsize=11, weight="bold", labelpad=8)
ax.set_axisbelow(True)
ax.set_ylim(0, y_max)
@@ -384,14 +427,15 @@ def draw_absolute(results: list[Result], field: str, ylabel: str,
for (_, mode), group in groupby(enumerate(selected), key=lambda item: (item[1][0].model, item[1][0].mode)):
group_positions = [positions[position] for position, _ in group]
centers.append((group_positions[0] + group_positions[-1]) / 2)
labels.append(mode.capitalize())
labels.append("Latency" if mode == "latency" else MODE_NAMES[mode])
ax.set_xticks(centers, labels, minor=True)
ax.tick_params(axis="x", which="minor", length=0, pad=4, labelsize=8)
ax.set_ylabel(ylabel, fontsize=10.5)
if arch == ARCHITECTURES[0]:
ax.set_ylabel(ylabel, fontsize=10.5)
if field == "throughput":
ax.ticklabel_format(axis="y", style="plain", useOffset=False)
ax.grid(axis="y", color="#E6EAF0", linewidth=0.8)
ax.tick_params(axis="y", length=0, pad=6)
ax.grid(axis="y")
ax.tick_params(axis="y", labelleft=arch == ARCHITECTURES[0], length=0, pad=6)
ax.tick_params(axis="x", which="major", length=0,
pad=22 if {row.mode for row, _ in selected} == {"latency", "throughput"} else 6)
@@ -399,19 +443,22 @@ def draw_absolute(results: list[Result], field: str, ylabel: str,
measurement = getattr(row, f"{compiler}_{field}") * scale
if not math.isfinite(measurement):
ax.text(x, y_max * 0.02, "N/A", rotation=90, ha="center",
va="bottom", fontsize=8, color="#85909F")
va="bottom", fontsize=8, color=TEXT_COLOR)
continue
status = getattr(row, f"{compiler}_validation")
pattern = STAGE_PATTERNS[row.pipeline] if compiler == "raptor" else ""
fill, border = COMPILER_STYLES[compiler]
ax.bar(x, measurement, width=1.0,
color=COMPILER_COLORS[compiler],
color=fill,
hatch=pattern,
edgecolor=TEXT_COLOR if pattern else "none",
linewidth=0.7 if pattern else 0.0, zorder=2)
edgecolor=border, linewidth=0.7, zorder=2)
label = (f"{measurement:,.2f}".rstrip("0").rstrip(".")
if field == "throughput" else f"{measurement:.3g}")
value_label(ax, x, measurement + y_max * 0.012, label, status)
value_label(ax, x, measurement + y_max * 0.012, label, status, invalid_label="P. INVALID")
fit_bar_widths(fig)
figure_legend(fig, results, field, absolute=True, raptor_only=raptor_only)
fit_x_tick_labels(fig)
fit_value_labels(fig)
save_figure(fig, basename, output_dir)
@@ -419,43 +466,50 @@ def draw_absolute(results: list[Result], field: str, ylabel: str,
def draw_pipeline_effect(results: list[Result], output_dir: Path) -> None:
rows = [row for row in results if row.mode == "throughput" or row.pipeline == 1]
stages = sorted({row.pipeline for row in rows})
panels = [(model, color) for model, color in zip(MODELS, COLORS)
if any(row.model == model for row in rows)]
panels = [model for model in MODELS if any(row.model == model for row in rows)]
lookup = {(row.arch, row.model, row.pipeline): row for row in rows}
rates = {key: quotient(1000, row.raptor_latency) if row.mode == "latency" else row.raptor_throughput
for key, row in lookup.items()}
fig, axes = plt.subplots(len(ARCHITECTURES), len(panels), sharex=True, sharey="col",
figsize=(3 * len(panels), 7.2), squeeze=False, layout="constrained")
for column, (model, color) in enumerate(panels):
figsize=(2.6 * len(panels), 4.3), squeeze=False, layout="constrained")
fig.supylabel("Throughput (inferences/s)", fontsize=10.5)
for column, model in enumerate(panels):
fill, border = NETWORK_STYLES[model]
for index, arch in enumerate(ARCHITECTURES):
ax = axes[index, column]
keys = [(arch, model, stage) for stage in stages]
values = [rates.get(key, math.nan) for key in keys]
valid = [key in lookup and lookup[key].raptor_validation == "PASS" for key in keys]
ax.plot(stages, values, color=color, linewidth=1.8)
ax.plot(stages, values, color=border, linewidth=1.8)
for stage, value, passed in zip(stages, values, valid):
if math.isfinite(value):
marker = ("s" if stage == 1 else "o") if passed else "x"
ax.plot(stage, value, color=color, marker=marker,
status = lookup[(arch, model, stage)].raptor_validation
ax.plot(stage, value, color=ERROR_COLOR if status == "FAIL" else border,
markerfacecolor=fill, marker=marker,
markersize=6, linestyle="none")
ax.set_xticks(stages)
ax.set_xlabel(model.upper(), fontsize=10.5, weight="bold", labelpad=8)
ax.grid(axis="y", color="#E6EAF0", linewidth=0.8)
ax.tick_params(axis="both", length=0, pad=6, labelbottom=True, labelleft=True)
if index == 0:
ax.set_title(MODEL_NAMES[model], fontsize=10.5, weight="normal", pad=8)
ax.grid(axis="y")
ax.tick_params(axis="both", length=0, pad=6,
labelbottom=index == len(ARCHITECTURES) - 1, labelleft=True)
ax.ticklabel_format(axis="y", style="plain", useOffset=False)
if column == 0:
ax.set_ylabel(f"{arch.replace('arch-', 'ARCHITECTURE ').upper()}\n"
"Throughput (inferences/s)", fontsize=10.5)
if column == len(panels) - 1:
ax.yaxis.set_label_position("right")
ax.set_ylabel(ARCH_NAMES[arch], fontsize=10.5, weight="bold", rotation=270, labelpad=24)
for ax in axes.flat:
ax.set_ylim(bottom=0)
handles = [Line2D([], [], color="#657081", marker=marker, linestyle="none", label=label)
for marker, label in (("s", "Derived as 1000ms / latency_ms"),
ax.margins(y=0.08)
handles = [Line2D([], [], color=TEXT_COLOR, markerfacecolor="white",
marker=marker, linestyle="none", label=label)
for marker, label in (("s", "Derived as 1 / latency"),
("o", "Throughput measurement"))]
if any(row.raptor_validation != "PASS" and math.isfinite(rates[key]) for key, row in lookup.items()):
handles.append(Line2D([], [], color="#424B58", marker="x", linestyle="none",
handles.append(Line2D([], [], color=TEXT_COLOR, marker="x", linestyle="none",
label="Raptor validation not PASS"))
fig.legend(handles=handles, loc="outside upper center", ncol=len(handles), frameon=False, fontsize=9.5)
add_legend(fig, handles)
fig.supxlabel("Raptor pipeline stages", fontsize=11)
fit_x_tick_labels(fig)
save_figure(fig, "raptor_pipeline_throughput", output_dir)
@@ -471,6 +525,7 @@ def main() -> None:
print(f"Results: {len(results)} configurations, {valid} valid comparisons, "
f"{len(results)-valid} comparisons without two PASS statuses")
draw_validation(results, args.output_dir)
draw_validation(results, args.output_dir, compact=True)
draw_pipeline_effect(results, args.output_dir)
yolo_only_models = {
model for model in MODELS if model.startswith("yolo")
@@ -480,12 +535,19 @@ def main() -> None:
}
comparisons = [row for row in results if row.model not in yolo_only_models]
yolo_only = [row for row in results if row.model in yolo_only_models]
compact_speedups = select_compact_speedups(comparisons)
for metric, label in (("speedup", "Speedup (x)"),
("energy", "Energy-per-inference advantage (x)"),
("power", "Average power advantage (x)")):
for suffix, diverging in (("comparison", False), ("diverging", True)):
draw_metric(compact_speedups, metric, label,
f"{metric}_compact_{suffix}", args.output_dir, diverging=diverging, compact=True)
for metric, name, label, mode in [
("latency_speedup", "longest_stage_latency", "Longest stage latency speedup (x)", None),
("latency_speedup", "latency", "Single-inference latency speedup (x)", "latency"),
("throughput_speedup", "throughput", "Throughput speedup (x)", "throughput"),
("power", "power", "Average power advantage (x)", None),
("energy", "energy", "Energy per inference advantage (x)", None),
("energy", "energy", "Energy-per-inference advantage (x)", None),
]:
selected = [row for row in comparisons if mode is None or row.mode == mode]
draw_metric(selected, metric, label,
+124 -22
View File
@@ -1,4 +1,4 @@
"""Shared appearance and PNG/SVG export for PIM result plots."""
"""Shared appearance and PNG/SVG/PDF export for PIM result plots."""
from pathlib import Path
from math import ceil
@@ -12,9 +12,32 @@ from matplotlib.patheffects import AbstractPathEffect
from matplotlib.transforms import Affine2D, IdentityTransform, TransformedPath
TEXT_COLOR = "#243449"
HATCH_COLOR = "#C62828"
COLORS = ("#2B65A1", "#168C8B", "#D18D32", "#8062A5", "#B65D60")
PALETTE = {
"sage": ("#B3D5CA", "#4B7C6A"),
"sand": ("#E1BC8B", "#856A44"),
"pale_blue": ("#EDF3FA", "#456B90"),
"blue": ("#B3CCE3", "#456B90"),
"pale_green": ("#EAF5F1", "#3E8271"),
"pale_sand": ("#FCF1E5", "#A87940"),
"red": ("#FF9494", "#C52020"),
"purple": ("#D1C0E0", "#77558F"),
}
TEXT_COLOR = "#222222"
MATRIX_TEXT_COLOR = "#1D2530"
ERROR_COLOR = PALETTE["red"][1]
GRID_COLOR = "#E6E9EC"
LINE_COLOR = "#C9CED3"
BAND_COLOR = "#F7F9FA"
COMPILER_STYLES = {"raptor": PALETTE["sage"], "pimcomp": PALETTE["sand"]}
METRIC_STYLES = {"latency": PALETTE["blue"], "throughput": PALETTE["sage"]}
NETWORK_STYLES = {"vgg8": PALETTE["blue"], "resnet18": PALETTE["sage"],
"resnet34": PALETTE["sand"], "googlenet": PALETTE["purple"],
"yolo11n": PALETTE["red"]}
COMPILER_NAMES = {"raptor": "Raptor", "pimcomp": "PIMCOMP"}
ARCH_NAMES = {"arch-a": "Arch-A", "arch-b": "Arch-B"}
MODEL_NAMES = {"vgg8": "VGG-8", "resnet18": "ResNet-18", "resnet34": "ResNet-34",
"googlenet": "GoogLeNet", "yolo11n": "YOLO11n"}
MODE_NAMES = {"latency": "Single-inference", "throughput": "Throughput"}
class CenteredHatch(AbstractPathEffect):
@@ -77,51 +100,130 @@ def configure_style() -> None:
"text.color": TEXT_COLOR,
"xtick.color": TEXT_COLOR,
"ytick.color": TEXT_COLOR,
"axes.edgecolor": "#CFD6DF",
"axes.edgecolor": LINE_COLOR,
"grid.color": GRID_COLOR,
"grid.linewidth": 0.8,
"hatch.color": TEXT_COLOR,
"savefig.facecolor": "white",
"figure.facecolor": "white",
})
def add_legend(fig, handles) -> None:
"""Use the fewest balanced legend rows that fit the figure width."""
renderer = fig.canvas.get_renderer()
for rows in range(1, len(handles) + 1):
columns = ceil(len(handles) / rows)
grid = []
start = 0
for row in range(rows):
count = len(handles) // rows + (row < len(handles) % rows)
grid.append(handles[start:start + count] + [None] * (columns - count))
start += count
ordered = [handle for column in zip(*grid) for handle in column]
legend = fig.legend(handles=[handle if handle is not None else Patch(visible=False) for handle in ordered],
labels=[handle.get_label() if handle is not None else "" for handle in ordered],
loc="outside upper center", ncols=columns, frameon=False, fontsize=9.5,
handlelength=1.7, columnspacing=1.5, handletextpad=0.6, labelspacing=0.4)
for handle, patch, text in zip(ordered, legend.legend_handles, legend.get_texts()):
if handle is None:
patch.set_visible(False)
text.set_visible(False)
if legend.get_window_extent(renderer).width <= fig.bbox.width - 20:
return
legend.remove()
def value_label(ax, x, y, label: str, status: str = "PASS", direction: int = 1,
fontsize: float = 8) -> None:
fontsize: float = 8, invalid_label: str = "INVALID") -> None:
alignment = "bottom" if direction > 0 else "top"
rotation = 90 if direction > 0 else -90
text = ax.text(x, y, label, rotation=rotation, ha="center", va=alignment,
fontsize=fontsize, color=TEXT_COLOR)
offset = direction * fontsize * 0.65 if status != "PASS" else 0
ax.annotate(label, (x, y), xytext=(-offset, 0), textcoords="offset points",
rotation=rotation, ha="center", va=alignment, fontsize=fontsize,
color=TEXT_COLOR)
if status != "PASS":
height = text.get_window_extent(ax.figure.canvas.get_renderer()).height
offset = direction * (height * 72 / ax.figure.dpi + 3)
ax.annotate("INVALID" if status == "FAIL" else status, (x, y),
xytext=(0, offset), textcoords="offset points",
ax.annotate(invalid_label if status == "FAIL" else status, (x, y),
xytext=(offset, 0), textcoords="offset points",
rotation=rotation, ha="center", va=alignment, fontsize=fontsize,
color=HATCH_COLOR)
color=ERROR_COLOR if status == "FAIL" else TEXT_COLOR)
def fit_x_tick_labels(fig) -> None:
"""Rotate only overlapping labels, clockwise by 30 degrees."""
fig.canvas.draw()
renderer = fig.canvas.get_renderer()
for minor in (False, True):
labels = [(text, text.get_window_extent(renderer))
for ax in fig.axes for text in ax.get_xticklabels(minor=minor)
if text.get_visible() and text.get_text()]
crowded = set()
for index, (text, bounds) in enumerate(labels):
for other_text, other_bounds in labels[:index]:
if bounds.overlaps(other_bounds):
crowded.update((text, other_text))
for text in crowded:
text.set_rotation(-30)
text.set_rotation_mode("anchor")
text.set_ha("right" if text.get_va() == "bottom" else "left")
fig.canvas.draw()
renderer = fig.canvas.get_renderer()
for ax in fig.axes:
bottom_minor = [text for text in ax.get_xticklabels(minor=True)
if text.get_va() == "top" and text.get_visible() and text.get_text()]
if bottom_minor:
height = max(text.get_window_extent(renderer).height for text in bottom_minor)
ax.tick_params(axis="x", which="major", pad=height * 72 / fig.dpi + 6)
fig.canvas.draw()
def fit_bar_widths(fig) -> None:
"""Reserve enough horizontal space for each bar's rendered annotations."""
fig.canvas.draw()
renderer = fig.canvas.get_renderer()
scale = 1.0
for ax in fig.axes:
labels = {}
for text in ax.texts:
if hasattr(text, "xy"):
labels.setdefault(text.xy[0], []).append(text.get_window_extent(renderer))
for container in ax.containers:
for bar in container.patches:
bounds = labels.get(bar.get_x() + bar.get_width() / 2, [])
if bounds:
width = max(box.x1 for box in bounds) - min(box.x0 for box in bounds)
scale = max(scale, (width + renderer.points_to_pixels(2)) / bar.get_window_extent(renderer).width)
if scale > 1:
fig.set_size_inches(fig.get_figwidth() * scale, fig.get_figheight())
def fit_value_labels(fig) -> None:
"""Reserve vertical space for rotated labels using their rendered size."""
"""Fit common vertical limits to the data and rendered value labels."""
for ax in fig.axes:
for text in ax.texts:
text.set_in_layout(False)
fig.canvas.draw()
renderer = fig.canvas.get_renderer()
limits = []
labels = []
for ax in fig.axes:
bounds = ax.get_window_extent(renderer)
labels = []
for text in ax.texts:
x, y = text.xy if hasattr(text, "xy") else text.get_position()
anchor = ax.transData.transform((x, y))[1]
extent = text.get_window_extent(renderer)
labels.append((y, max(0, anchor - extent.y0 + 3) / bounds.height,
max(0, extent.y1 - anchor + 3) / bounds.height))
lower, upper = ax.get_ylim()
for _ in range(12):
lower = min([lower, *((y - below * upper) / (1 - below) for y, below, _ in labels)])
upper = max([upper, *((y - above * lower) / (1 - above) for y, _, above in labels)])
limits.append((lower, upper))
limits.append(ax.dataLim.intervaly if ax.has_data() else ax.get_ylim())
lower = min(0, *(lower for lower, _ in limits))
upper = max(0, *(upper for _, upper in limits))
if lower == upper:
upper = lower + 1
for _ in range(12):
lower = min([lower, *((y - below * upper) / (1 - below) for y, below, _ in labels)])
upper = max([upper, *((y - above * lower) / (1 - above) for y, _, above in labels)])
for ax in fig.axes:
ax.set_ylim(min(lower for lower, _ in limits), max(upper for _, upper in limits))
ax.set_ylim(lower, upper)
def save_figure(fig, basename: str, output_dir: Path) -> None:
@@ -129,7 +231,7 @@ def save_figure(fig, basename: str, output_dir: Path) -> None:
hatch = patch.get_hatch() or ""
if any(symbol in hatch for symbol in "./\\"):
patch.set_path_effects([CenteredHatch()])
for extension in ("png", "svg"):
for extension in ("png", "svg", "pdf"):
target = output_dir / extension / f"{basename}.{extension}"
target.parent.mkdir(parents=True, exist_ok=True)
fig.savefig(target, dpi=220, bbox_inches="tight", pad_inches=0.04)