This commit is contained in:
@@ -47,27 +47,9 @@ guarantees, so those variants may fail numerical validation and exit nonzero.
|
||||
Baseline and `no-plan` must pass. Percentages from failing variants are not
|
||||
valid speedups; deadline averages do not prove steady-state convergence.
|
||||
|
||||
## Plots
|
||||
## Result data
|
||||
|
||||
```bash
|
||||
.venv/bin/python validation/tools/pim/ablation/generate_plots.py
|
||||
# For a custom experiment root:
|
||||
.venv/bin/python validation/tools/pim/ablation/generate_plots.py \
|
||||
--input /path/to/experiment/results_ablation.csv
|
||||
```
|
||||
|
||||
Requires Matplotlib from the root `requirements.txt`. By default, reads the
|
||||
canonical CSV beside this script. Plots go to
|
||||
`plots/png/` and `plots/svg/` beside the selected CSV; override with
|
||||
`--output-dir PATH`.
|
||||
Single-variant comparison CSVs are not accepted.
|
||||
|
||||
The checked-in CSV is an intentional result dataset; plotting uses its current
|
||||
contents without running benchmarks. Invalid reciprocal percentage pairs render
|
||||
as `N/A`; individual legitimate extreme changes are preserved. See
|
||||
[performance availability](../../../../docs/evaluation/methodology.md#functional-validity-and-performance-availability).
|
||||
|
||||
Plots separate modes and pipelines. Throughput-mode latency means output
|
||||
interval. Failed validation is labeled `INVALID`, other non-PASS statuses keep
|
||||
their names, and missing measurements show `N/A`. See the
|
||||
The checked-in `results_ablation.csv` is an intentional result dataset for tabular
|
||||
reporting. Ablation experiments and aggregation remain available through the
|
||||
runner above. Percentages from failing variants are not valid speedups. See the
|
||||
[comparison metric definitions](../../../../docs/evaluation/methodology.md#metric-interpretation).
|
||||
|
||||
@@ -1,156 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Plot percentage changes from an experiment's combined results_ablation.csv."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import math
|
||||
import sys
|
||||
import warnings
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
from plot_style import COLORS, configure_style, save_figure, value_label
|
||||
|
||||
import matplotlib.pyplot as plt
|
||||
from matplotlib.patches import Patch
|
||||
from matplotlib.ticker import PercentFormatter
|
||||
|
||||
from run_ablation import ABLATION_DEFAULT_MODELS, DEFAULT_RESULTS_PATH, RESULT_FIELDS, VARIANTS
|
||||
|
||||
|
||||
VARIANT_COLORS = {variant.name: color for variant, color in zip(VARIANTS, COLORS)}
|
||||
STATUSES = ("PASS", "FAIL", "SKIPPED", "NA")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Result:
|
||||
variant: str
|
||||
arch: str
|
||||
model: str
|
||||
mode: str
|
||||
pipeline: int
|
||||
validation: str
|
||||
metrics: dict[str, float]
|
||||
|
||||
|
||||
def percentage(value: str) -> float:
|
||||
if value == "NA":
|
||||
return math.nan
|
||||
if not value.endswith("%"):
|
||||
raise ValueError(f"Expected a percentage or NA, got {value!r}")
|
||||
parsed = float(value[:-1])
|
||||
if not math.isfinite(parsed):
|
||||
raise ValueError(f"Non-finite percentage: {value!r}")
|
||||
return parsed
|
||||
|
||||
|
||||
def load_results(path: Path) -> list[Result]:
|
||||
rows = []
|
||||
seen = set()
|
||||
invalid_pairs = []
|
||||
with path.open(newline="", encoding="utf-8-sig") as stream:
|
||||
reader = csv.DictReader(stream)
|
||||
if not {"variant", *RESULT_FIELDS} <= set(reader.fieldnames or []):
|
||||
raise ValueError("Expected the combined ablation CSV with variant and percentage columns; "
|
||||
"a per-variant results_comparison.csv cannot be used here.")
|
||||
for row in reader:
|
||||
variant, mode, status = row["variant"], row["mode"], row["functional_validation"]
|
||||
if variant not in VARIANT_COLORS or mode not in ("latency", "throughput") or status not in STATUSES:
|
||||
raise ValueError(f"Unknown ablation variant, mode, or validation status: {variant}, {mode}, {status}")
|
||||
pipeline = int(row["raptor_pipeline"])
|
||||
key = (variant, row["arch"], row["model"], mode, pipeline)
|
||||
if key in seen:
|
||||
raise ValueError(f"Duplicate ablation configuration: {key}")
|
||||
seen.add(key)
|
||||
metrics = {metric: percentage(row[metric])
|
||||
for metric in ("latency", "throughput", "power", "energy")}
|
||||
# Each may round to -100% alone; reciprocal metrics cannot both do so.
|
||||
if mode == "throughput" and metrics["latency"] == metrics["throughput"] == -100:
|
||||
metrics["latency"] = metrics["throughput"] = math.nan
|
||||
invalid_pairs.append(key)
|
||||
rows.append(Result(variant, row["arch"], row["model"], mode, pipeline, status, metrics))
|
||||
if not rows:
|
||||
raise ValueError("The ablation CSV contains no results")
|
||||
if invalid_pairs:
|
||||
warnings.warn(f"{len(invalid_pairs)} invalid reciprocal latency/throughput percentage pairs; "
|
||||
f"displaying N/A. Examples: {invalid_pairs[:3]}", stacklevel=2)
|
||||
return rows
|
||||
|
||||
|
||||
def draw_metric(results: list[Result], metric: str, ylabel: str, output_dir: Path) -> None:
|
||||
architectures = sorted({row.arch for row in results})
|
||||
present_models = {row.model for row in results}
|
||||
models = [model for model in ABLATION_DEFAULT_MODELS if model in present_models]
|
||||
models += sorted(present_models - set(models))
|
||||
variants = [name for name in VARIANT_COLORS if any(row.variant == name for row in results)]
|
||||
lookup = {(row.arch, row.model, row.variant): row for row in results}
|
||||
values = [row.metrics[metric] for row in results if math.isfinite(row.metrics[metric])]
|
||||
lower, upper = min(0, min(values, default=0)), max(0, max(values, default=0))
|
||||
padding = max(upper - lower, 1) * 0.35
|
||||
fig, axes = plt.subplots(1, len(architectures), sharey=True, squeeze=False,
|
||||
figsize=(7 * len(architectures), 5.2), layout="constrained")
|
||||
handles = [Patch(facecolor=VARIANT_COLORS[name], label=name) for name in variants]
|
||||
if len(values) < len(architectures) * len(models) * len(variants):
|
||||
handles.append(Patch(facecolor="white", edgecolor="#B2BAC5", label="N/A: missing measurement"))
|
||||
fig.legend(handles=handles, loc="outside upper center", ncol=4, frameon=False, fontsize=9.5)
|
||||
width = 0.8 / len(variants)
|
||||
mode, pipeline = results[0].mode, results[0].pipeline
|
||||
for ax, arch in zip(axes[0], architectures):
|
||||
ax.set_axisbelow(True)
|
||||
ax.set_ylim(lower - padding if lower < 0 else 0, upper + padding)
|
||||
ax.set_xlim(-0.5, len(models) - 0.5)
|
||||
ax.set_xticks(range(len(models)), [model.upper() for model in models], fontsize=9.5)
|
||||
ax.set_xlabel(f"{arch.replace('arch-', 'ARCHITECTURE ').upper()}\n"
|
||||
f"{mode.capitalize()}, {pipeline} stage" + ("s" if pipeline != 1 else ""),
|
||||
fontsize=11, weight="bold", labelpad=8)
|
||||
ax.set_ylabel(ylabel, fontsize=10.5)
|
||||
ax.yaxis.set_major_formatter(PercentFormatter(xmax=100, decimals=0))
|
||||
ax.tick_params(axis="both", length=0, pad=6, labelleft=True)
|
||||
ax.grid(axis="y", color="#E6EAF0", linewidth=0.8)
|
||||
ax.axhline(0, color="#536478", linewidth=1.2, linestyle="--")
|
||||
for index, model in enumerate(models):
|
||||
if index % 2 == 0:
|
||||
ax.axvspan(index - 0.5, index + 0.5, facecolor="#F7F9FC", zorder=0)
|
||||
for offset, variant in enumerate(variants):
|
||||
x = index + (offset - (len(variants) - 1) / 2) * width
|
||||
row = lookup.get((arch, model, variant))
|
||||
value = row.metrics[metric] if row else math.nan
|
||||
if not math.isfinite(value):
|
||||
ax.text(x, padding * 0.05, "N/A", rotation=90, ha="center", va="bottom",
|
||||
fontsize=8, color="#85909F")
|
||||
continue
|
||||
ax.bar(x, value, width=width * 0.9, color=VARIANT_COLORS[variant],
|
||||
edgecolor="none", linewidth=0, zorder=2)
|
||||
direction = -1 if value < 0 else 1
|
||||
value_label(ax, x, value + direction * padding * 0.05, f"{value:+,.2f}%",
|
||||
row.validation, direction)
|
||||
save_figure(fig, f"{mode}_pipeline{pipeline}_{metric}", output_dir)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--input", type=Path, default=DEFAULT_RESULTS_PATH)
|
||||
parser.add_argument("--output-dir", type=Path)
|
||||
args = parser.parse_args()
|
||||
if args.output_dir is None:
|
||||
args.output_dir = args.input.parent / "plots"
|
||||
configure_style()
|
||||
results = load_results(args.input)
|
||||
print(f"Results: {len(results)} configurations, "
|
||||
f"{sum(row.validation != 'PASS' for row in results)} without PASS validation")
|
||||
for mode, pipeline in sorted({(row.mode, row.pipeline) for row in results}):
|
||||
rows = [row for row in results if row.mode == mode and row.pipeline == pipeline]
|
||||
for metric, label in (
|
||||
("latency", "Single-inference latency" if mode == "latency" else "Output interval"),
|
||||
("throughput", "Throughput"), ("power", "Average power"), ("energy", "Energy per inference"),
|
||||
):
|
||||
if metric == "throughput" and mode == "latency":
|
||||
continue
|
||||
draw_metric(rows, metric, f"{label} change from baseline (%)", args.output_dir)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -72,7 +72,7 @@ The comparator defaults to absolute/relative tolerances `1e-3`/`1e-4`;
|
||||
|
||||
Requires Matplotlib from the root `requirements.txt`. Defaults to the supplied
|
||||
`results_comparison.csv` beside the script and writes `plots/png/` and
|
||||
`plots/svg/` there. Use `--input PATH` for a runner-generated CSV and
|
||||
`plots/svg/` and `plots/pdf/` there. Use `--input PATH` for a runner-generated CSV and
|
||||
`--output-dir PATH` to override the destination. The plotter requires the full
|
||||
40-case matrix: five models × Arch-A/B × latency pipeline 1 and throughput
|
||||
pipelines 2/4/8. Duplicate, missing, unexpected, or inconsistent shared Pimcomp
|
||||
@@ -81,6 +81,9 @@ configurations are rejected before plots are written.
|
||||
| Plot family | Meaning |
|
||||
|---|---|
|
||||
| `validation_status` | Recorded functional status for each compiler and experiment. |
|
||||
| `validation_status_compact` | Compact functional status matrix. |
|
||||
| `speedup_compact_*` | Touching single-inference latency and best-pipeline throughput bars for each network. |
|
||||
| `energy_compact_*`, `power_compact_*` | Energy and power advantages for the same cases selected by the compact speedup plots. |
|
||||
| `latency_*` | Single-inference latency, latency mode only. |
|
||||
| `throughput_*` | Inferences/s, throughput mode only. |
|
||||
| `longest_stage_latency_*` | Combined single-inference latency and throughput output interval; the latter is a stage-latency proxy. |
|
||||
@@ -93,7 +96,28 @@ advantages in either direction. Ratios above 1 favor Raptor: Pimcomp/Raptor for
|
||||
latency, power, and energy; Raptor/Pimcomp for throughput. Diverging bars use
|
||||
winning factor minus one, with parity at zero.
|
||||
|
||||
Failed validation is labeled `INVALID`; other non-PASS statuses keep their
|
||||
The shared `plot_style.py` defines the manuscript palette and display names.
|
||||
Raptor uses sage green; PIMCOMP uses plain sand. Stage patterns,
|
||||
outlines, marker shapes, and status text preserve distinctions in grayscale.
|
||||
Compact plots use blue for the single-inference case and hatched sage
|
||||
green for the throughput case.
|
||||
Each architecture/network pair uses the highest recorded throughput across pipelines.
|
||||
Energy and power use those same cases, without selecting a different pipeline for each metric.
|
||||
Validation labels remain visible for the selected cases. Units appear in axis
|
||||
titles and bar values; axis tick labels contain numbers only.
|
||||
|
||||
Throughput-stage bars touch within each network in both absolute and advantage
|
||||
plots. Bar-label widths are measured before export; dense figures expand only
|
||||
as needed to contain values and invalidity annotations. Legends use the available
|
||||
width and balance entries across rows when wrapping is necessary.
|
||||
|
||||
Comparison panels share y-axis titles and tick labels. Each network in the
|
||||
pipeline plot retains its own scale, with y-axis ticks on both architecture rows.
|
||||
Only the bottom row shows the shared x-axis ticks. Network names appear above
|
||||
the top row; architecture names appear on the right. That plot
|
||||
trims unused vertical range and is intended to span both manuscript columns.
|
||||
|
||||
Failed validation is labeled `P. INVALID`; other non-PASS statuses keep their
|
||||
names, and missing measurements show `N/A`. Relative bars require both
|
||||
compilers to pass to be valid. Repeated Pimcomp batch measurements and statuses
|
||||
must agree before being collapsed. `FAIL` alone does not identify the cause.
|
||||
|
||||
@@ -1,25 +1,25 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate publication-ready Raptor vs Pimcomp comparison charts.
|
||||
"""Generate publication-ready Raptor vs PIMCOMP comparison charts.
|
||||
|
||||
Usage:
|
||||
.venv/bin/python validation/tools/pim/pimcomp/compare/generate_plots.py
|
||||
|
||||
Dependencies: matplotlib (no pandas or seaborn required).
|
||||
By default, reads results_comparison.csv beside this script and writes plots/
|
||||
beside it, with outputs grouped in png/ and svg/ under the output directory.
|
||||
beside it, with outputs grouped in png/, svg/, and pdf/ under the output directory.
|
||||
|
||||
YOLO models without Pimcomp measurements get separate Raptor-only charts.
|
||||
YOLO models without PIMCOMP measurements get separate Raptor-only charts.
|
||||
Comparisons use vertical bars for ratios, diverging advantages, and absolute
|
||||
values, with compiler colors and patterns for the Raptor pipeline stages.
|
||||
Throughput-mode latency is the output interval, a bottleneck-stage proxy,
|
||||
not end-to-end inference latency. Separate throughput plots use inferences/s.
|
||||
Energy is reported per inference; power is averaged over the simulation.
|
||||
Numerical results whose compiler did not pass functional validation have a red
|
||||
status following their value label; missing Pimcomp
|
||||
status following their value label; missing PIMCOMP
|
||||
measurements appear as 'N/A'. The fixed compiler palette is shared by
|
||||
all metrics. Relative plots use Pimcomp / Raptor for latency, power and energy,
|
||||
and Raptor / Pimcomp for throughput, so >1 favors Raptor. Diverging plots put the winner's
|
||||
factor above (Raptor) or below (Pimcomp) parity, with 1x at the zero line.
|
||||
all metrics. Relative plots use PIMCOMP / Raptor for latency, power and energy,
|
||||
and Raptor / PIMCOMP for throughput, so >1 favors Raptor. Diverging plots put the winner's
|
||||
factor above (Raptor) or below (PIMCOMP) parity, with 1x at the zero line.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -39,20 +39,22 @@ from matplotlib.lines import Line2D
|
||||
from matplotlib.ticker import FuncFormatter, MultipleLocator
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
from plot_style import COLORS, HATCH_COLOR, TEXT_COLOR, configure_style, fit_value_labels, save_figure, value_label
|
||||
from plot_style import (ARCH_NAMES, BAND_COLOR, COMPILER_NAMES, COMPILER_STYLES,
|
||||
ERROR_COLOR, LINE_COLOR, MATRIX_TEXT_COLOR, METRIC_STYLES,
|
||||
MODE_NAMES, MODEL_NAMES, NETWORK_STYLES,
|
||||
PALETTE, TEXT_COLOR, add_legend, configure_style, fit_value_labels,
|
||||
fit_bar_widths, fit_x_tick_labels, save_figure, value_label)
|
||||
|
||||
|
||||
COMPILER_COLORS = {"raptor": COLORS[0], "pimcomp": COLORS[2]}
|
||||
STAGE_PATTERNS = {1: "", 2: "..", 4: "//", 8: "\\\\"}
|
||||
ARCHITECTURES = ("arch-a", "arch-b")
|
||||
MODELS = ("vgg8", "resnet18", "resnet34", "googlenet", "yolo11n")
|
||||
CONFIGURATIONS = (("latency", 1), ("throughput", 2), ("throughput", 4), ("throughput", 8))
|
||||
LINE_COLOR = "#A8B2C0"
|
||||
VALIDATION_STYLES = {
|
||||
"PASS": ("#168C8B", "white", ""),
|
||||
"FAIL": ("#D18D32", HATCH_COLOR, ""),
|
||||
"SKIPPED": ("#E7DFF0", TEXT_COLOR, ""),
|
||||
"NA": ("#E8EDF1", TEXT_COLOR, ""),
|
||||
"PASS": PALETTE["sage"],
|
||||
"FAIL": PALETTE["red"],
|
||||
"SKIPPED": PALETTE["pale_sand"],
|
||||
"NA": PALETTE["pale_blue"],
|
||||
}
|
||||
|
||||
|
||||
@@ -85,6 +87,10 @@ class Result:
|
||||
def throughput_speedup(self) -> float:
|
||||
return quotient(self.raptor_throughput, self.pimcomp_throughput)
|
||||
|
||||
@property
|
||||
def speedup(self) -> float:
|
||||
return self.latency_speedup if self.mode == "latency" else self.throughput_speedup
|
||||
|
||||
|
||||
def positive_float(text: str | None) -> float:
|
||||
try:
|
||||
@@ -129,7 +135,7 @@ def load_results(path: Path) -> list[Result]:
|
||||
seen[key] = line
|
||||
schedule = row["pimcomp_pipeline"].strip().lower()
|
||||
if schedule != ("element" if mode == "latency" else "batch"):
|
||||
raise ValueError(f"{key}: incompatible Pimcomp schedule {schedule!r}")
|
||||
raise ValueError(f"{key}: incompatible PIMCOMP schedule {schedule!r}")
|
||||
rows.append(Result(
|
||||
arch=arch,
|
||||
model=model,
|
||||
@@ -185,20 +191,20 @@ def figure_legend(fig, results: list[Result], field: str,
|
||||
for _, compiler, value in available} - {None}
|
||||
if len(visible_compilers) > 1:
|
||||
for compiler in compilers:
|
||||
handles.append(Patch(facecolor=COMPILER_COLORS[compiler],
|
||||
label=compiler.capitalize() + ("" if absolute else " advantage")))
|
||||
fill, border = COMPILER_STYLES[compiler]
|
||||
handles.append(Patch(facecolor=fill, edgecolor=border,
|
||||
label=COMPILER_NAMES[compiler] + ("" if absolute else " advantage")))
|
||||
handles += [Patch(facecolor="white", edgecolor=TEXT_COLOR, hatch=STAGE_PATTERNS[depth],
|
||||
label=f"{depth} stage" + (" (latency)" if depth == 1 else "s"))
|
||||
label=("1 stage (latency)" if absolute else "Single-inference")
|
||||
if depth == 1 else f"{depth} stages")
|
||||
for depth in sorted(stages) if stages != {1}]
|
||||
styles = []
|
||||
if len(available) < len(measurements):
|
||||
styles.append(Patch(facecolor="white", edgecolor="#B2BAC5",
|
||||
styles.append(Patch(facecolor="white", edgecolor=LINE_COLOR,
|
||||
label="N/A: missing measurement"))
|
||||
handles += styles
|
||||
if handles:
|
||||
fig.legend(handles=handles, loc="outside upper center", ncol=(len(handles) + 1) // 2,
|
||||
frameon=False, fontsize=9.5, handlelength=1.7, columnspacing=1.5,
|
||||
handletextpad=0.6, labelspacing=0.4)
|
||||
add_legend(fig, handles)
|
||||
|
||||
|
||||
def absolute_columns(results: list[Result], raptor_only: bool = False) -> list[tuple[Result, str]]:
|
||||
@@ -218,7 +224,7 @@ def absolute_columns(results: list[Result], raptor_only: bool = False) -> list[t
|
||||
for row in throughput
|
||||
}
|
||||
if len(signatures) != 1:
|
||||
raise ValueError(f"{arch}/{model}: inconsistent shared Pimcomp batch results across pipelines")
|
||||
raise ValueError(f"{arch}/{model}: inconsistent shared PIMCOMP batch results across pipelines")
|
||||
columns.append((throughput[0], "pimcomp"))
|
||||
return columns
|
||||
|
||||
@@ -231,34 +237,50 @@ def label_groups(ax, results: list[Result], positions: list[float] | None = None
|
||||
for index, (model, rows) in enumerate(groupby(results, key=lambda row: row.model)):
|
||||
end = start + len(list(rows))
|
||||
centers.append((positions[start] + positions[end - 1]) / 2)
|
||||
labels.append(model.upper())
|
||||
labels.append(MODEL_NAMES[model])
|
||||
left = ax.get_xlim()[0] if start == 0 else (positions[start - 1] + positions[start]) / 2
|
||||
right = ax.get_xlim()[1] if end == len(results) else (positions[end - 1] + positions[end]) / 2
|
||||
if index % 2 == 0:
|
||||
ax.axvspan(left, right, facecolor="#F7F9FC", zorder=0)
|
||||
ax.axvspan(left, right, facecolor=BAND_COLOR, zorder=0)
|
||||
if index > 0:
|
||||
ax.axvline(left, color="#DDE3EB", linewidth=0.9)
|
||||
ax.axvline(left, color=LINE_COLOR, linewidth=0.7)
|
||||
start = end
|
||||
ax.set_xticks(centers, labels, fontsize=9.5)
|
||||
|
||||
|
||||
def grouped_positions(results: list[Result]) -> list[float]:
|
||||
positions = []
|
||||
for index, row in enumerate(results):
|
||||
if index == 0:
|
||||
positions.append(0.0)
|
||||
continue
|
||||
previous = results[index - 1]
|
||||
gap = 0.8 if row.model != previous.model else 0.4 if row.mode != previous.mode else 0.0
|
||||
positions.append(positions[-1] + 1.0 + gap)
|
||||
return positions
|
||||
|
||||
|
||||
def diverging_ratio(ratio: float) -> float:
|
||||
return ratio - 1 if ratio >= 1 else 1 - 1 / ratio
|
||||
|
||||
|
||||
def draw_validation(results: list[Result], output_dir: Path) -> None:
|
||||
columns = absolute_columns(results)
|
||||
configurations = list(dict.fromkeys((row.mode, row.pipeline if compiler == "raptor" else row.pimcomp_schedule,
|
||||
compiler) for row, compiler in columns))
|
||||
def draw_validation(results: list[Result], output_dir: Path, compact: bool = False) -> None:
|
||||
cells = {}
|
||||
for row, compiler in absolute_columns(results):
|
||||
stage = row.pipeline if compiler == "raptor" else row.pimcomp_schedule
|
||||
key = (row.arch, row.model, row.mode, None if compact else stage, compiler)
|
||||
cells.setdefault(key, set()).add(getattr(row, f"{compiler}_validation"))
|
||||
configurations = list(dict.fromkeys(key[2:] for key in cells))
|
||||
positions = []
|
||||
for index, (mode, _, _) in enumerate(configurations):
|
||||
gap = 0.4 if index and mode != configurations[index - 1][0] else 0
|
||||
positions.append(positions[-1] + 1 + gap if index else 0)
|
||||
configuration_x = dict(zip(configurations, positions))
|
||||
headers = [f"Raptor\n{stage} stages" if mode == "throughput" and compiler == "raptor"
|
||||
else "Pimcomp\nbatch" if mode == "throughput" else compiler.capitalize()
|
||||
headers = [f"Raptor\n{stage} stages" if mode == "throughput" and compiler == "raptor" and not compact
|
||||
else "PIMCOMP\nbatch" if mode == "throughput" and compiler == "pimcomp" and not compact
|
||||
else COMPILER_NAMES[compiler]
|
||||
for mode, stage, compiler in configurations]
|
||||
fig, axes = plt.subplots(1, 2, figsize=(14, 3.8), layout="constrained")
|
||||
fig, axes = plt.subplots(1, 2, figsize=(8.5 if compact else 12, 3.3), layout="constrained")
|
||||
for ax, arch in zip(axes, ARCHITECTURES):
|
||||
ax.set_xlim(positions[0] - 0.55, positions[-1] + 0.55)
|
||||
ax.set_ylim(len(MODELS) - 0.5, -0.5)
|
||||
@@ -266,31 +288,41 @@ def draw_validation(results: list[Result], output_dir: Path) -> None:
|
||||
for mode, group in groupby(zip(configurations, positions), key=lambda item: item[0][0]):
|
||||
group_positions = [x for _, x in group]
|
||||
centers.append((group_positions[0] + group_positions[-1]) / 2)
|
||||
modes.append(mode.capitalize())
|
||||
modes.append("Latency" if mode == "latency" else MODE_NAMES[mode])
|
||||
ax.set_xticks(centers, modes, fontsize=10)
|
||||
ax.set_xticks(positions, headers, minor=True)
|
||||
ax.tick_params(axis="x", which="minor", top=True, labeltop=True,
|
||||
bottom=False, labelbottom=False, length=0, pad=6, labelsize=9)
|
||||
ax.set_yticks(range(len(MODELS)), [model.upper() for model in MODELS], fontsize=9)
|
||||
ax.set_yticks(range(len(MODELS)), [MODEL_NAMES[model] for model in MODELS], fontsize=9)
|
||||
ax.tick_params(axis="both", length=0, pad=6)
|
||||
architecture = "ARCHITECTURE A" if arch == "arch-a" else "ARCHITECTURE B"
|
||||
ax.tick_params(axis="y", labelleft=arch == ARCHITECTURES[0])
|
||||
architecture = ARCH_NAMES[arch]
|
||||
ax.set_xlabel(architecture, fontsize=11, weight="bold", labelpad=8)
|
||||
for row, compiler in columns:
|
||||
if row.arch != arch:
|
||||
for (cell_arch, model, mode, stage, compiler), statuses in cells.items():
|
||||
if cell_arch != arch:
|
||||
continue
|
||||
key = (row.mode, row.pipeline if compiler == "raptor" else row.pimcomp_schedule, compiler)
|
||||
x, y = configuration_x[key], MODELS.index(row.model)
|
||||
status = getattr(row, f"{compiler}_validation")
|
||||
color, text_color, _ = VALIDATION_STYLES[status]
|
||||
x, y = configuration_x[(mode, stage, compiler)], MODELS.index(model)
|
||||
status = next(status for status in ("FAIL", "SKIPPED", "NA", "PASS") if status in statuses)
|
||||
color, border = VALIDATION_STYLES[status]
|
||||
ax.add_patch(Rectangle((x - 0.5, y - 0.42), 1, 0.84,
|
||||
facecolor=color, edgecolor="#657081", linewidth=0.5))
|
||||
facecolor=color, edgecolor=border, linewidth=0.7))
|
||||
ax.text(x, y, "N/A" if status == "NA" else status,
|
||||
color=text_color, ha="center", va="center", fontsize=9)
|
||||
save_figure(fig, "validation_status", output_dir)
|
||||
color=MATRIX_TEXT_COLOR, ha="center", va="center", fontsize=12, weight="normal")
|
||||
fit_x_tick_labels(fig)
|
||||
save_figure(fig, "validation_status_compact" if compact else "validation_status", output_dir)
|
||||
|
||||
|
||||
def select_compact_speedups(results: list[Result]) -> list[Result]:
|
||||
selected = []
|
||||
for _, group in groupby(results, key=lambda row: (row.arch, row.model, row.mode)):
|
||||
rows = list(group)
|
||||
selected.append(max(rows, key=lambda row: row.speedup if math.isfinite(row.speedup) else -math.inf))
|
||||
return selected
|
||||
|
||||
|
||||
def draw_metric(results: list[Result], field: str, ylabel: str,
|
||||
basename: str, output_dir: Path, diverging: bool = False) -> None:
|
||||
basename: str, output_dir: Path, diverging: bool = False,
|
||||
compact: bool = False) -> None:
|
||||
data = [diverging_ratio(getattr(row, field)) if diverging else getattr(row, field)
|
||||
for row in results]
|
||||
available = [v for v in data if math.isfinite(v)]
|
||||
@@ -301,49 +333,68 @@ def draw_metric(results: list[Result], field: str, ylabel: str,
|
||||
y_min = math.floor((lower - padding) * 2) / 2 if diverging else 0
|
||||
y_interval = 1.0 if y_max - y_min > 4.5 else 0.5
|
||||
|
||||
fig, axes = plt.subplots(1, 2, sharey=True, figsize=(14, 5.2),
|
||||
fig, axes = plt.subplots(1, 2, sharey=True, figsize=(8.5 if compact else 12, 4.4),
|
||||
layout="constrained")
|
||||
figure_legend(fig, results, field)
|
||||
if compact:
|
||||
single_metric = "latency" if field == "speedup" else field
|
||||
throughput_suffix = "" if field == "speedup" else f" {field}"
|
||||
handles = [Patch(facecolor=METRIC_STYLES["latency"][0],
|
||||
edgecolor=METRIC_STYLES["latency"][1],
|
||||
label=f"Single-inference {single_metric}"),
|
||||
Patch(facecolor=METRIC_STYLES["throughput"][0],
|
||||
edgecolor=METRIC_STYLES["throughput"][1], hatch="//",
|
||||
label=f"Best-case throughput{throughput_suffix}")]
|
||||
|
||||
for ax, arch in zip(axes, ARCHITECTURES):
|
||||
selected = [row for row in results if row.arch == arch]
|
||||
ax.set_xlabel("ARCHITECTURE A" if arch == "arch-a" else "ARCHITECTURE B",
|
||||
positions = [index + index // 2 for index in range(len(selected))] if compact else grouped_positions(selected)
|
||||
ax.set_xlabel(ARCH_NAMES[arch],
|
||||
fontsize=11, weight="bold", labelpad=8)
|
||||
ax.set_axisbelow(True)
|
||||
ax.set_ylim(y_min, y_max)
|
||||
ax.set_xlim(-0.5, len(selected) - 0.5)
|
||||
label_groups(ax, selected)
|
||||
ax.tick_params(axis="y", labelleft=True, pad=5, length=0)
|
||||
ax.set_xlim(-0.9, positions[-1] + 0.9 if positions else 0.9)
|
||||
label_groups(ax, selected, positions)
|
||||
ax.tick_params(axis="y", labelleft=arch == ARCHITECTURES[0], pad=5, length=0)
|
||||
ax.tick_params(axis="x", labelsize=9.5, length=0, pad=6)
|
||||
ax.yaxis.set_major_locator(MultipleLocator(y_interval))
|
||||
if diverging:
|
||||
ax.yaxis.set_major_formatter(FuncFormatter(lambda value, _: f"{abs(value) + 1:g}×"))
|
||||
ax.grid(axis="y", color="#E6EAF0", linewidth=0.8)
|
||||
ax.axhline(0 if diverging else 1, color="#536478", linewidth=1.2,
|
||||
ax.yaxis.set_major_formatter(FuncFormatter(lambda value, _: f"{abs(value) + 1:g}"))
|
||||
ax.grid(axis="y")
|
||||
ax.axhline(0 if diverging else 1, color=TEXT_COLOR, linewidth=1.0,
|
||||
linestyle="--", zorder=3)
|
||||
ax.set_ylabel(ylabel, fontsize=10.5, labelpad=6)
|
||||
if arch == ARCHITECTURES[0]:
|
||||
ax.set_ylabel(ylabel, fontsize=10.5, labelpad=6)
|
||||
|
||||
for position, row in enumerate(selected):
|
||||
for position, row in zip(positions, selected):
|
||||
ratio = getattr(row, field)
|
||||
if not math.isfinite(ratio):
|
||||
ax.text(position, (y_max - y_min) * 0.02, "N/A", ha="center", va="bottom",
|
||||
fontsize=9.5, color="#85909F", fontstyle="italic")
|
||||
fontsize=9.5, color=TEXT_COLOR, fontstyle="italic")
|
||||
continue
|
||||
compiler = advantage_compiler(ratio)
|
||||
statuses = [status for status in (row.raptor_validation, row.pimcomp_validation) if status != "PASS"]
|
||||
status = "FAIL" if "FAIL" in statuses else next(iter(statuses), "PASS")
|
||||
pattern = STAGE_PATTERNS[row.pipeline]
|
||||
color, border = COMPILER_STYLES[compiler] if compiler else ("white", TEXT_COLOR)
|
||||
if compact:
|
||||
pattern = "//" if row.mode == "throughput" else ""
|
||||
color, border = METRIC_STYLES[row.mode]
|
||||
measurement = diverging_ratio(ratio) if diverging else ratio
|
||||
ax.bar(position, measurement, width=0.72,
|
||||
color=COMPILER_COLORS.get(compiler, "#657081"),
|
||||
ax.bar(position, measurement, width=1.0,
|
||||
color=color,
|
||||
hatch=pattern,
|
||||
edgecolor=TEXT_COLOR if pattern else "none",
|
||||
linewidth=0.7 if pattern else 0.0, zorder=2)
|
||||
edgecolor=border, linewidth=0.7, zorder=2)
|
||||
direction = -1 if measurement < 0 else 1
|
||||
factor = max(ratio, 1 / ratio) if diverging else ratio
|
||||
value_label(ax, position, measurement + direction * (y_max - y_min) * 0.012,
|
||||
f"{factor:.2f}×", status, direction, fontsize=9.1)
|
||||
f"{factor:.2f}×", status, direction, fontsize=9.1, invalid_label="P. INVALID")
|
||||
|
||||
fit_bar_widths(fig)
|
||||
if compact:
|
||||
add_legend(fig, handles)
|
||||
else:
|
||||
figure_legend(fig, results, field)
|
||||
fit_x_tick_labels(fig)
|
||||
fit_value_labels(fig)
|
||||
save_figure(fig, basename, output_dir)
|
||||
|
||||
@@ -356,21 +407,13 @@ def draw_absolute(results: list[Result], field: str, ylabel: str,
|
||||
for row, compiler in columns
|
||||
if math.isfinite(getattr(row, f"{compiler}_{field}"))]
|
||||
y_max = max(available, default=1.0) * 1.35
|
||||
fig, axes = plt.subplots(1, 2, sharey=True, figsize=(14, 5.2),
|
||||
fig, axes = plt.subplots(1, 2, sharey=True, figsize=(12, 4.4),
|
||||
layout="constrained")
|
||||
figure_legend(fig, results, field, absolute=True, raptor_only=raptor_only)
|
||||
|
||||
for ax, arch in zip(axes, ARCHITECTURES):
|
||||
selected = [(row, compiler) for row, compiler in columns if row.arch == arch]
|
||||
positions = []
|
||||
for index, (row, _) in enumerate(selected):
|
||||
if index == 0:
|
||||
positions.append(0.0)
|
||||
continue
|
||||
previous = selected[index - 1][0]
|
||||
gap = 0.8 if row.model != previous.model else 0.4 if row.mode != previous.mode else 0.0
|
||||
positions.append(positions[-1] + 1.0 + gap)
|
||||
ax.set_xlabel("ARCHITECTURE A" if arch == "arch-a" else "ARCHITECTURE B",
|
||||
positions = grouped_positions([row for row, _ in selected])
|
||||
ax.set_xlabel(ARCH_NAMES[arch],
|
||||
fontsize=11, weight="bold", labelpad=8)
|
||||
ax.set_axisbelow(True)
|
||||
ax.set_ylim(0, y_max)
|
||||
@@ -384,14 +427,15 @@ def draw_absolute(results: list[Result], field: str, ylabel: str,
|
||||
for (_, mode), group in groupby(enumerate(selected), key=lambda item: (item[1][0].model, item[1][0].mode)):
|
||||
group_positions = [positions[position] for position, _ in group]
|
||||
centers.append((group_positions[0] + group_positions[-1]) / 2)
|
||||
labels.append(mode.capitalize())
|
||||
labels.append("Latency" if mode == "latency" else MODE_NAMES[mode])
|
||||
ax.set_xticks(centers, labels, minor=True)
|
||||
ax.tick_params(axis="x", which="minor", length=0, pad=4, labelsize=8)
|
||||
ax.set_ylabel(ylabel, fontsize=10.5)
|
||||
if arch == ARCHITECTURES[0]:
|
||||
ax.set_ylabel(ylabel, fontsize=10.5)
|
||||
if field == "throughput":
|
||||
ax.ticklabel_format(axis="y", style="plain", useOffset=False)
|
||||
ax.grid(axis="y", color="#E6EAF0", linewidth=0.8)
|
||||
ax.tick_params(axis="y", length=0, pad=6)
|
||||
ax.grid(axis="y")
|
||||
ax.tick_params(axis="y", labelleft=arch == ARCHITECTURES[0], length=0, pad=6)
|
||||
ax.tick_params(axis="x", which="major", length=0,
|
||||
pad=22 if {row.mode for row, _ in selected} == {"latency", "throughput"} else 6)
|
||||
|
||||
@@ -399,19 +443,22 @@ def draw_absolute(results: list[Result], field: str, ylabel: str,
|
||||
measurement = getattr(row, f"{compiler}_{field}") * scale
|
||||
if not math.isfinite(measurement):
|
||||
ax.text(x, y_max * 0.02, "N/A", rotation=90, ha="center",
|
||||
va="bottom", fontsize=8, color="#85909F")
|
||||
va="bottom", fontsize=8, color=TEXT_COLOR)
|
||||
continue
|
||||
status = getattr(row, f"{compiler}_validation")
|
||||
pattern = STAGE_PATTERNS[row.pipeline] if compiler == "raptor" else ""
|
||||
fill, border = COMPILER_STYLES[compiler]
|
||||
ax.bar(x, measurement, width=1.0,
|
||||
color=COMPILER_COLORS[compiler],
|
||||
color=fill,
|
||||
hatch=pattern,
|
||||
edgecolor=TEXT_COLOR if pattern else "none",
|
||||
linewidth=0.7 if pattern else 0.0, zorder=2)
|
||||
edgecolor=border, linewidth=0.7, zorder=2)
|
||||
label = (f"{measurement:,.2f}".rstrip("0").rstrip(".")
|
||||
if field == "throughput" else f"{measurement:.3g}")
|
||||
value_label(ax, x, measurement + y_max * 0.012, label, status)
|
||||
value_label(ax, x, measurement + y_max * 0.012, label, status, invalid_label="P. INVALID")
|
||||
|
||||
fit_bar_widths(fig)
|
||||
figure_legend(fig, results, field, absolute=True, raptor_only=raptor_only)
|
||||
fit_x_tick_labels(fig)
|
||||
fit_value_labels(fig)
|
||||
save_figure(fig, basename, output_dir)
|
||||
|
||||
@@ -419,43 +466,50 @@ def draw_absolute(results: list[Result], field: str, ylabel: str,
|
||||
def draw_pipeline_effect(results: list[Result], output_dir: Path) -> None:
|
||||
rows = [row for row in results if row.mode == "throughput" or row.pipeline == 1]
|
||||
stages = sorted({row.pipeline for row in rows})
|
||||
panels = [(model, color) for model, color in zip(MODELS, COLORS)
|
||||
if any(row.model == model for row in rows)]
|
||||
panels = [model for model in MODELS if any(row.model == model for row in rows)]
|
||||
lookup = {(row.arch, row.model, row.pipeline): row for row in rows}
|
||||
rates = {key: quotient(1000, row.raptor_latency) if row.mode == "latency" else row.raptor_throughput
|
||||
for key, row in lookup.items()}
|
||||
fig, axes = plt.subplots(len(ARCHITECTURES), len(panels), sharex=True, sharey="col",
|
||||
figsize=(3 * len(panels), 7.2), squeeze=False, layout="constrained")
|
||||
for column, (model, color) in enumerate(panels):
|
||||
figsize=(2.6 * len(panels), 4.3), squeeze=False, layout="constrained")
|
||||
fig.supylabel("Throughput (inferences/s)", fontsize=10.5)
|
||||
for column, model in enumerate(panels):
|
||||
fill, border = NETWORK_STYLES[model]
|
||||
for index, arch in enumerate(ARCHITECTURES):
|
||||
ax = axes[index, column]
|
||||
keys = [(arch, model, stage) for stage in stages]
|
||||
values = [rates.get(key, math.nan) for key in keys]
|
||||
valid = [key in lookup and lookup[key].raptor_validation == "PASS" for key in keys]
|
||||
ax.plot(stages, values, color=color, linewidth=1.8)
|
||||
ax.plot(stages, values, color=border, linewidth=1.8)
|
||||
for stage, value, passed in zip(stages, values, valid):
|
||||
if math.isfinite(value):
|
||||
marker = ("s" if stage == 1 else "o") if passed else "x"
|
||||
ax.plot(stage, value, color=color, marker=marker,
|
||||
status = lookup[(arch, model, stage)].raptor_validation
|
||||
ax.plot(stage, value, color=ERROR_COLOR if status == "FAIL" else border,
|
||||
markerfacecolor=fill, marker=marker,
|
||||
markersize=6, linestyle="none")
|
||||
ax.set_xticks(stages)
|
||||
ax.set_xlabel(model.upper(), fontsize=10.5, weight="bold", labelpad=8)
|
||||
ax.grid(axis="y", color="#E6EAF0", linewidth=0.8)
|
||||
ax.tick_params(axis="both", length=0, pad=6, labelbottom=True, labelleft=True)
|
||||
if index == 0:
|
||||
ax.set_title(MODEL_NAMES[model], fontsize=10.5, weight="normal", pad=8)
|
||||
ax.grid(axis="y")
|
||||
ax.tick_params(axis="both", length=0, pad=6,
|
||||
labelbottom=index == len(ARCHITECTURES) - 1, labelleft=True)
|
||||
ax.ticklabel_format(axis="y", style="plain", useOffset=False)
|
||||
if column == 0:
|
||||
ax.set_ylabel(f"{arch.replace('arch-', 'ARCHITECTURE ').upper()}\n"
|
||||
"Throughput (inferences/s)", fontsize=10.5)
|
||||
if column == len(panels) - 1:
|
||||
ax.yaxis.set_label_position("right")
|
||||
ax.set_ylabel(ARCH_NAMES[arch], fontsize=10.5, weight="bold", rotation=270, labelpad=24)
|
||||
for ax in axes.flat:
|
||||
ax.set_ylim(bottom=0)
|
||||
handles = [Line2D([], [], color="#657081", marker=marker, linestyle="none", label=label)
|
||||
for marker, label in (("s", "Derived as 1000ms / latency_ms"),
|
||||
ax.margins(y=0.08)
|
||||
handles = [Line2D([], [], color=TEXT_COLOR, markerfacecolor="white",
|
||||
marker=marker, linestyle="none", label=label)
|
||||
for marker, label in (("s", "Derived as 1 / latency"),
|
||||
("o", "Throughput measurement"))]
|
||||
if any(row.raptor_validation != "PASS" and math.isfinite(rates[key]) for key, row in lookup.items()):
|
||||
handles.append(Line2D([], [], color="#424B58", marker="x", linestyle="none",
|
||||
handles.append(Line2D([], [], color=TEXT_COLOR, marker="x", linestyle="none",
|
||||
label="Raptor validation not PASS"))
|
||||
fig.legend(handles=handles, loc="outside upper center", ncol=len(handles), frameon=False, fontsize=9.5)
|
||||
add_legend(fig, handles)
|
||||
fig.supxlabel("Raptor pipeline stages", fontsize=11)
|
||||
fit_x_tick_labels(fig)
|
||||
save_figure(fig, "raptor_pipeline_throughput", output_dir)
|
||||
|
||||
|
||||
@@ -471,6 +525,7 @@ def main() -> None:
|
||||
print(f"Results: {len(results)} configurations, {valid} valid comparisons, "
|
||||
f"{len(results)-valid} comparisons without two PASS statuses")
|
||||
draw_validation(results, args.output_dir)
|
||||
draw_validation(results, args.output_dir, compact=True)
|
||||
draw_pipeline_effect(results, args.output_dir)
|
||||
yolo_only_models = {
|
||||
model for model in MODELS if model.startswith("yolo")
|
||||
@@ -480,12 +535,19 @@ def main() -> None:
|
||||
}
|
||||
comparisons = [row for row in results if row.model not in yolo_only_models]
|
||||
yolo_only = [row for row in results if row.model in yolo_only_models]
|
||||
compact_speedups = select_compact_speedups(comparisons)
|
||||
for metric, label in (("speedup", "Speedup (x)"),
|
||||
("energy", "Energy-per-inference advantage (x)"),
|
||||
("power", "Average power advantage (x)")):
|
||||
for suffix, diverging in (("comparison", False), ("diverging", True)):
|
||||
draw_metric(compact_speedups, metric, label,
|
||||
f"{metric}_compact_{suffix}", args.output_dir, diverging=diverging, compact=True)
|
||||
for metric, name, label, mode in [
|
||||
("latency_speedup", "longest_stage_latency", "Longest stage latency speedup (x)", None),
|
||||
("latency_speedup", "latency", "Single-inference latency speedup (x)", "latency"),
|
||||
("throughput_speedup", "throughput", "Throughput speedup (x)", "throughput"),
|
||||
("power", "power", "Average power advantage (x)", None),
|
||||
("energy", "energy", "Energy per inference advantage (x)", None),
|
||||
("energy", "energy", "Energy-per-inference advantage (x)", None),
|
||||
]:
|
||||
selected = [row for row in comparisons if mode is None or row.mode == mode]
|
||||
draw_metric(selected, metric, label,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Shared appearance and PNG/SVG export for PIM result plots."""
|
||||
"""Shared appearance and PNG/SVG/PDF export for PIM result plots."""
|
||||
|
||||
from pathlib import Path
|
||||
from math import ceil
|
||||
@@ -12,9 +12,32 @@ from matplotlib.patheffects import AbstractPathEffect
|
||||
from matplotlib.transforms import Affine2D, IdentityTransform, TransformedPath
|
||||
|
||||
|
||||
TEXT_COLOR = "#243449"
|
||||
HATCH_COLOR = "#C62828"
|
||||
COLORS = ("#2B65A1", "#168C8B", "#D18D32", "#8062A5", "#B65D60")
|
||||
PALETTE = {
|
||||
"sage": ("#B3D5CA", "#4B7C6A"),
|
||||
"sand": ("#E1BC8B", "#856A44"),
|
||||
"pale_blue": ("#EDF3FA", "#456B90"),
|
||||
"blue": ("#B3CCE3", "#456B90"),
|
||||
"pale_green": ("#EAF5F1", "#3E8271"),
|
||||
"pale_sand": ("#FCF1E5", "#A87940"),
|
||||
"red": ("#FF9494", "#C52020"),
|
||||
"purple": ("#D1C0E0", "#77558F"),
|
||||
}
|
||||
TEXT_COLOR = "#222222"
|
||||
MATRIX_TEXT_COLOR = "#1D2530"
|
||||
ERROR_COLOR = PALETTE["red"][1]
|
||||
GRID_COLOR = "#E6E9EC"
|
||||
LINE_COLOR = "#C9CED3"
|
||||
BAND_COLOR = "#F7F9FA"
|
||||
COMPILER_STYLES = {"raptor": PALETTE["sage"], "pimcomp": PALETTE["sand"]}
|
||||
METRIC_STYLES = {"latency": PALETTE["blue"], "throughput": PALETTE["sage"]}
|
||||
NETWORK_STYLES = {"vgg8": PALETTE["blue"], "resnet18": PALETTE["sage"],
|
||||
"resnet34": PALETTE["sand"], "googlenet": PALETTE["purple"],
|
||||
"yolo11n": PALETTE["red"]}
|
||||
COMPILER_NAMES = {"raptor": "Raptor", "pimcomp": "PIMCOMP"}
|
||||
ARCH_NAMES = {"arch-a": "Arch-A", "arch-b": "Arch-B"}
|
||||
MODEL_NAMES = {"vgg8": "VGG-8", "resnet18": "ResNet-18", "resnet34": "ResNet-34",
|
||||
"googlenet": "GoogLeNet", "yolo11n": "YOLO11n"}
|
||||
MODE_NAMES = {"latency": "Single-inference", "throughput": "Throughput"}
|
||||
|
||||
|
||||
class CenteredHatch(AbstractPathEffect):
|
||||
@@ -77,51 +100,130 @@ def configure_style() -> None:
|
||||
"text.color": TEXT_COLOR,
|
||||
"xtick.color": TEXT_COLOR,
|
||||
"ytick.color": TEXT_COLOR,
|
||||
"axes.edgecolor": "#CFD6DF",
|
||||
"axes.edgecolor": LINE_COLOR,
|
||||
"grid.color": GRID_COLOR,
|
||||
"grid.linewidth": 0.8,
|
||||
"hatch.color": TEXT_COLOR,
|
||||
"savefig.facecolor": "white",
|
||||
"figure.facecolor": "white",
|
||||
})
|
||||
|
||||
|
||||
def add_legend(fig, handles) -> None:
|
||||
"""Use the fewest balanced legend rows that fit the figure width."""
|
||||
renderer = fig.canvas.get_renderer()
|
||||
for rows in range(1, len(handles) + 1):
|
||||
columns = ceil(len(handles) / rows)
|
||||
grid = []
|
||||
start = 0
|
||||
for row in range(rows):
|
||||
count = len(handles) // rows + (row < len(handles) % rows)
|
||||
grid.append(handles[start:start + count] + [None] * (columns - count))
|
||||
start += count
|
||||
ordered = [handle for column in zip(*grid) for handle in column]
|
||||
legend = fig.legend(handles=[handle if handle is not None else Patch(visible=False) for handle in ordered],
|
||||
labels=[handle.get_label() if handle is not None else "" for handle in ordered],
|
||||
loc="outside upper center", ncols=columns, frameon=False, fontsize=9.5,
|
||||
handlelength=1.7, columnspacing=1.5, handletextpad=0.6, labelspacing=0.4)
|
||||
for handle, patch, text in zip(ordered, legend.legend_handles, legend.get_texts()):
|
||||
if handle is None:
|
||||
patch.set_visible(False)
|
||||
text.set_visible(False)
|
||||
if legend.get_window_extent(renderer).width <= fig.bbox.width - 20:
|
||||
return
|
||||
legend.remove()
|
||||
|
||||
|
||||
def value_label(ax, x, y, label: str, status: str = "PASS", direction: int = 1,
|
||||
fontsize: float = 8) -> None:
|
||||
fontsize: float = 8, invalid_label: str = "INVALID") -> None:
|
||||
alignment = "bottom" if direction > 0 else "top"
|
||||
rotation = 90 if direction > 0 else -90
|
||||
text = ax.text(x, y, label, rotation=rotation, ha="center", va=alignment,
|
||||
fontsize=fontsize, color=TEXT_COLOR)
|
||||
offset = direction * fontsize * 0.65 if status != "PASS" else 0
|
||||
ax.annotate(label, (x, y), xytext=(-offset, 0), textcoords="offset points",
|
||||
rotation=rotation, ha="center", va=alignment, fontsize=fontsize,
|
||||
color=TEXT_COLOR)
|
||||
if status != "PASS":
|
||||
height = text.get_window_extent(ax.figure.canvas.get_renderer()).height
|
||||
offset = direction * (height * 72 / ax.figure.dpi + 3)
|
||||
ax.annotate("INVALID" if status == "FAIL" else status, (x, y),
|
||||
xytext=(0, offset), textcoords="offset points",
|
||||
ax.annotate(invalid_label if status == "FAIL" else status, (x, y),
|
||||
xytext=(offset, 0), textcoords="offset points",
|
||||
rotation=rotation, ha="center", va=alignment, fontsize=fontsize,
|
||||
color=HATCH_COLOR)
|
||||
color=ERROR_COLOR if status == "FAIL" else TEXT_COLOR)
|
||||
|
||||
|
||||
def fit_x_tick_labels(fig) -> None:
|
||||
"""Rotate only overlapping labels, clockwise by 30 degrees."""
|
||||
fig.canvas.draw()
|
||||
renderer = fig.canvas.get_renderer()
|
||||
for minor in (False, True):
|
||||
labels = [(text, text.get_window_extent(renderer))
|
||||
for ax in fig.axes for text in ax.get_xticklabels(minor=minor)
|
||||
if text.get_visible() and text.get_text()]
|
||||
crowded = set()
|
||||
for index, (text, bounds) in enumerate(labels):
|
||||
for other_text, other_bounds in labels[:index]:
|
||||
if bounds.overlaps(other_bounds):
|
||||
crowded.update((text, other_text))
|
||||
for text in crowded:
|
||||
text.set_rotation(-30)
|
||||
text.set_rotation_mode("anchor")
|
||||
text.set_ha("right" if text.get_va() == "bottom" else "left")
|
||||
fig.canvas.draw()
|
||||
renderer = fig.canvas.get_renderer()
|
||||
for ax in fig.axes:
|
||||
bottom_minor = [text for text in ax.get_xticklabels(minor=True)
|
||||
if text.get_va() == "top" and text.get_visible() and text.get_text()]
|
||||
if bottom_minor:
|
||||
height = max(text.get_window_extent(renderer).height for text in bottom_minor)
|
||||
ax.tick_params(axis="x", which="major", pad=height * 72 / fig.dpi + 6)
|
||||
fig.canvas.draw()
|
||||
|
||||
|
||||
def fit_bar_widths(fig) -> None:
|
||||
"""Reserve enough horizontal space for each bar's rendered annotations."""
|
||||
fig.canvas.draw()
|
||||
renderer = fig.canvas.get_renderer()
|
||||
scale = 1.0
|
||||
for ax in fig.axes:
|
||||
labels = {}
|
||||
for text in ax.texts:
|
||||
if hasattr(text, "xy"):
|
||||
labels.setdefault(text.xy[0], []).append(text.get_window_extent(renderer))
|
||||
for container in ax.containers:
|
||||
for bar in container.patches:
|
||||
bounds = labels.get(bar.get_x() + bar.get_width() / 2, [])
|
||||
if bounds:
|
||||
width = max(box.x1 for box in bounds) - min(box.x0 for box in bounds)
|
||||
scale = max(scale, (width + renderer.points_to_pixels(2)) / bar.get_window_extent(renderer).width)
|
||||
if scale > 1:
|
||||
fig.set_size_inches(fig.get_figwidth() * scale, fig.get_figheight())
|
||||
|
||||
|
||||
def fit_value_labels(fig) -> None:
|
||||
"""Reserve vertical space for rotated labels using their rendered size."""
|
||||
"""Fit common vertical limits to the data and rendered value labels."""
|
||||
for ax in fig.axes:
|
||||
for text in ax.texts:
|
||||
text.set_in_layout(False)
|
||||
fig.canvas.draw()
|
||||
renderer = fig.canvas.get_renderer()
|
||||
limits = []
|
||||
labels = []
|
||||
for ax in fig.axes:
|
||||
bounds = ax.get_window_extent(renderer)
|
||||
labels = []
|
||||
for text in ax.texts:
|
||||
x, y = text.xy if hasattr(text, "xy") else text.get_position()
|
||||
anchor = ax.transData.transform((x, y))[1]
|
||||
extent = text.get_window_extent(renderer)
|
||||
labels.append((y, max(0, anchor - extent.y0 + 3) / bounds.height,
|
||||
max(0, extent.y1 - anchor + 3) / bounds.height))
|
||||
lower, upper = ax.get_ylim()
|
||||
for _ in range(12):
|
||||
lower = min([lower, *((y - below * upper) / (1 - below) for y, below, _ in labels)])
|
||||
upper = max([upper, *((y - above * lower) / (1 - above) for y, _, above in labels)])
|
||||
limits.append((lower, upper))
|
||||
limits.append(ax.dataLim.intervaly if ax.has_data() else ax.get_ylim())
|
||||
lower = min(0, *(lower for lower, _ in limits))
|
||||
upper = max(0, *(upper for _, upper in limits))
|
||||
if lower == upper:
|
||||
upper = lower + 1
|
||||
for _ in range(12):
|
||||
lower = min([lower, *((y - below * upper) / (1 - below) for y, below, _ in labels)])
|
||||
upper = max([upper, *((y - above * lower) / (1 - above) for y, _, above in labels)])
|
||||
for ax in fig.axes:
|
||||
ax.set_ylim(min(lower for lower, _ in limits), max(upper for _, upper in limits))
|
||||
ax.set_ylim(lower, upper)
|
||||
|
||||
|
||||
def save_figure(fig, basename: str, output_dir: Path) -> None:
|
||||
@@ -129,7 +231,7 @@ def save_figure(fig, basename: str, output_dir: Path) -> None:
|
||||
hatch = patch.get_hatch() or ""
|
||||
if any(symbol in hatch for symbol in "./\\"):
|
||||
patch.set_path_effects([CenteredHatch()])
|
||||
for extension in ("png", "svg"):
|
||||
for extension in ("png", "svg", "pdf"):
|
||||
target = output_dir / extension / f"{basename}.{extension}"
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
fig.savefig(target, dpi=220, bbox_inches="tight", pad_inches=0.04)
|
||||
|
||||
Reference in New Issue
Block a user