NiccoloN
a77edafcbf
Validate Operations / validate-operations (push) Has been cancelled
plot adjustments validation adjustments
971 lines
51 KiB
Python
971 lines
51 KiB
Python
from __future__ import annotations
|
||
|
||
import csv
|
||
from dataclasses import replace
|
||
import importlib.util
|
||
import math
|
||
from pathlib import Path
|
||
import subprocess
|
||
import sys
|
||
|
||
import pytest
|
||
|
||
|
||
TOOLS = Path(__file__).resolve().parents[1] / "tools/pim"
|
||
|
||
|
||
@pytest.fixture
|
||
def comparison(monkeypatch):
|
||
directory = TOOLS / "pimcomp/compare"
|
||
monkeypatch.syspath_prepend(str(directory))
|
||
spec = importlib.util.spec_from_file_location("comparison_plots", directory / "generate_plots.py")
|
||
module = importlib.util.module_from_spec(spec)
|
||
monkeypatch.setitem(sys.modules, spec.name, module)
|
||
spec.loader.exec_module(module)
|
||
return module
|
||
|
||
|
||
@pytest.fixture
|
||
def comparison_csv():
|
||
with (TOOLS / "pimcomp/compare/results_comparison.csv").open(newline="") as stream:
|
||
reader = csv.DictReader(stream)
|
||
return reader.fieldnames, list(reader)
|
||
|
||
|
||
def write_csv(path, fields, rows):
|
||
with path.open("w", newline="") as stream:
|
||
writer = csv.DictWriter(stream, fieldnames=fields)
|
||
writer.writeheader()
|
||
writer.writerows(rows)
|
||
return path
|
||
|
||
|
||
@pytest.mark.parametrize("custom", [False, True])
|
||
def test_comparison_plot_cli_input_paths(comparison, tmp_path, monkeypatch, custom):
|
||
expected = tmp_path / "custom.csv" if custom else TOOLS / "pimcomp/compare/results_comparison.csv"
|
||
seen = []
|
||
|
||
class InputCaptured(Exception):
|
||
pass
|
||
|
||
def capture(path):
|
||
seen.append(path)
|
||
raise InputCaptured
|
||
|
||
monkeypatch.setattr(comparison, "load_results", capture)
|
||
monkeypatch.setattr(comparison, "configure_style", lambda **kwargs: None)
|
||
monkeypatch.setattr(sys, "argv", ["generate_plots.py", *(["--input", str(expected)] if custom else [])])
|
||
with pytest.raises(InputCaptured):
|
||
comparison.main()
|
||
assert seen == [expected]
|
||
|
||
|
||
@pytest.mark.parametrize("custom", [False, True])
|
||
def test_comparison_plot_cli_output_paths(comparison, tmp_path, monkeypatch, custom):
|
||
expected = tmp_path / "custom" if custom else TOOLS / "pimcomp/compare/plots/default"
|
||
seen = []
|
||
|
||
class OutputCaptured(Exception):
|
||
pass
|
||
|
||
def capture(rows, output):
|
||
seen.append(output)
|
||
raise OutputCaptured
|
||
|
||
monkeypatch.setattr(comparison, "draw_validation", capture)
|
||
monkeypatch.setattr(sys, "argv", ["generate_plots.py", *(["--output-dir", str(expected)] if custom else [])])
|
||
with pytest.raises(OutputCaptured):
|
||
comparison.main()
|
||
assert seen == [expected]
|
||
|
||
|
||
@pytest.mark.parametrize("flags,colors,anonymize", [([], True, False),
|
||
(["--colors", "--no-anonymize"], True, False), (["--no-colors"], False, False),
|
||
(["--anonymize"], True, True), (["--no-colors", "--anonymize"], False, True),
|
||
(["--no-colors", "--colors", "--anonymize", "--no-anonymize"], True, False)])
|
||
def test_plot_display_flags(comparison, monkeypatch, flags, colors, anonymize):
|
||
captured = []
|
||
|
||
class StyleCaptured(Exception):
|
||
pass
|
||
|
||
def capture(**options):
|
||
captured.append(options)
|
||
raise StyleCaptured
|
||
|
||
monkeypatch.setattr(comparison, "configure_style", capture)
|
||
monkeypatch.setattr(sys, "argv", ["generate_plots.py", *flags])
|
||
with pytest.raises(StyleCaptured):
|
||
comparison.main()
|
||
assert captured == [{"colors": colors, "anonymize": anonymize}]
|
||
|
||
|
||
@pytest.mark.parametrize("flags,borders", [([], True), (["--marker-borders"], True),
|
||
(["--no-marker-borders"], False), (["--no-marker-borders", "--marker-borders"], True)])
|
||
def test_marker_border_cli_flags(comparison, monkeypatch, flags, borders):
|
||
captured = []
|
||
|
||
def capture(*args, marker_borders):
|
||
captured.append(marker_borders)
|
||
|
||
for name in ("draw_validation", "draw_metric", "draw_absolute"):
|
||
monkeypatch.setattr(comparison, name, lambda *args, **kwargs: None)
|
||
for name in ("draw_compact_summary", "draw_pipeline_effect"):
|
||
monkeypatch.setattr(comparison, name, capture)
|
||
monkeypatch.setattr(sys, "argv", ["generate_plots.py", *flags])
|
||
comparison.main()
|
||
assert captured == [borders, borders]
|
||
|
||
|
||
@pytest.mark.parametrize("borders", [True, False])
|
||
def test_marker_borders_reach_data_legends_and_svg(comparison, tmp_path, monkeypatch, borders):
|
||
from xml.etree import ElementTree as ET
|
||
from matplotlib.lines import Line2D
|
||
from matplotlib.colors import to_rgba
|
||
|
||
comparison.configure_style()
|
||
rows = comparison.load_results(TOOLS / "pimcomp/compare/results_comparison.csv")
|
||
save = comparison.save_figure
|
||
|
||
def capture(fig, name, output):
|
||
try:
|
||
size = 6.5 if name == "performance_energy_compact" else 7
|
||
for ax in fig.axes:
|
||
points = [line for line in ax.lines if line.get_marker() in ("o", "s", "^", "D")]
|
||
assert points
|
||
for point in points:
|
||
assert point.get_markersize() == size
|
||
assert (point.get_markeredgewidth() > 0) == borders
|
||
assert to_rgba(point.get_markerfacecolor())[3] == 1
|
||
if not borders:
|
||
assert to_rgba(point.get_markerfacecolor()) == to_rgba(point.get_markeredgecolor())
|
||
handles = [handle for handle in fig.legends[0].legend_handles if isinstance(handle, Line2D)]
|
||
assert handles
|
||
for handle in handles:
|
||
assert handle.get_markersize() == size
|
||
assert (handle.get_markeredgewidth() > 0) == borders
|
||
if not borders:
|
||
assert to_rgba(handle.get_markerfacecolor()) not in ((1, 1, 1, 1), (0, 0, 0, 0))
|
||
assert to_rgba(handle.get_markerfacecolor()) == to_rgba(handle.get_markeredgecolor())
|
||
save(fig, name, output)
|
||
finally:
|
||
comparison.plt.close(fig)
|
||
root = ET.parse(output / f"svg/{name}.svg")
|
||
paths = [node for node in root.findall('.//{http://www.w3.org/2000/svg}defs/{http://www.w3.org/2000/svg}path')
|
||
if node.get("id", "").startswith("m")]
|
||
assert paths
|
||
assert all(("stroke:" in node.get("style", "")) == borders for node in paths)
|
||
|
||
monkeypatch.setattr(comparison, "save_figure", capture)
|
||
comparison.draw_compact_summary(rows, tmp_path, marker_borders=borders)
|
||
comparison.draw_pipeline_effect(rows, tmp_path, marker_borders=borders)
|
||
|
||
|
||
@pytest.mark.parametrize("colors", [True, False])
|
||
@pytest.mark.parametrize("anonymize", [True, False])
|
||
def test_plot_display_variant_exports(comparison, tmp_path, colors, anonymize):
|
||
import re
|
||
from PIL import Image, ImageChops
|
||
|
||
try:
|
||
comparison.configure_style(colors=colors, anonymize=anonymize)
|
||
rows = comparison.load_results(TOOLS / "pimcomp/compare/results_comparison.csv")
|
||
comparison.draw_compact_summary(rows, tmp_path)
|
||
selected = comparison.select_compact_speedups([row for row in rows if row.model != "yolo11n"])
|
||
comparison.draw_metric(selected, "speedup", "Speedup (x)", "speedup_compact_comparison",
|
||
tmp_path, compact=True)
|
||
for name in ("performance_energy_compact", "speedup_compact_comparison"):
|
||
svg = (tmp_path / f"svg/{name}.svg").read_text()
|
||
if name == "performance_energy_compact":
|
||
assert ("Anonimized better" in svg) == anonymize
|
||
assert ("Raptor better" in svg) == (not anonymize)
|
||
svg_colors = re.findall(r"(?:fill|stroke):\s*#([0-9a-fA-F]{6})", svg)
|
||
assert svg_colors
|
||
assert any(color[:2] != color[2:4] or color[2:4] != color[4:] for color in svg_colors) == colors
|
||
with Image.open(tmp_path / f"png/{name}.png") as picture:
|
||
red, green, blue = picture.convert("RGB").split()
|
||
chromatic = (ImageChops.difference(red, green).getbbox() is not None or
|
||
ImageChops.difference(green, blue).getbbox() is not None)
|
||
assert chromatic == colors
|
||
assert (tmp_path / f"pdf/{name}.pdf").read_bytes().startswith(b"%PDF-")
|
||
finally:
|
||
comparison.configure_style()
|
||
|
||
|
||
def test_anonymization_reaches_all_plot_labels(comparison, monkeypatch):
|
||
from matplotlib.text import Text
|
||
|
||
captured = {}
|
||
|
||
def capture(fig, name, output):
|
||
captured[name] = [text.get_text() for text in fig.findobj(Text)]
|
||
comparison.plt.close(fig)
|
||
|
||
monkeypatch.setattr(comparison, "save_figure", capture)
|
||
monkeypatch.setattr(sys, "argv", ["generate_plots.py", "--anonymize"])
|
||
try:
|
||
comparison.main()
|
||
assert len(captured) == 30
|
||
assert all("Raptor" not in label for labels in captured.values() for label in labels)
|
||
assert "Anonimized pipeline stages" in captured["raptor_pipeline_throughput"]
|
||
assert "Anonimized better →" in captured["performance_energy_compact"]
|
||
assert "Anonimized\n2 stages" in captured["validation_status"]
|
||
finally:
|
||
comparison.configure_style()
|
||
|
||
|
||
def test_exact_comparison_configuration_matrix(comparison, comparison_csv, tmp_path):
|
||
fields, rows = comparison_csv
|
||
results = comparison.load_results(write_csv(tmp_path / "results.csv", fields, rows))
|
||
keys = {(row.arch, row.model, row.mode, row.pipeline) for row in results}
|
||
assert keys == {(arch, model, mode, stage) for arch in comparison.ARCHITECTURES
|
||
for model in comparison.MODELS for mode, stage in comparison.CONFIGURATIONS}
|
||
|
||
|
||
@pytest.mark.parametrize("damage,message", [("missing", "Missing"), ("duplicate", "Duplicate"),
|
||
("arch", "Unexpected"), ("model", "Unexpected"),
|
||
("pipeline", "Unexpected"), ("mode", "Unexpected")])
|
||
def test_comparison_rejects_bad_matrix_keys(comparison, comparison_csv, tmp_path, damage, message):
|
||
fields, rows = comparison_csv
|
||
if damage == "missing":
|
||
rows.pop()
|
||
elif damage == "duplicate":
|
||
rows[-1] = dict(rows[-2]) # Still 40 rows and 20 per architecture.
|
||
else:
|
||
field = "raptor_pipeline" if damage == "pipeline" else damage
|
||
rows[-1][field] = {"arch": "arch-c", "model": "other", "pipeline": "3", "mode": "other"}[damage]
|
||
with pytest.raises(ValueError, match=message):
|
||
comparison.load_results(write_csv(tmp_path / "results.csv", fields, rows))
|
||
|
||
|
||
def test_matrix_validation_survives_optimized_python(comparison_csv, tmp_path):
|
||
fields, rows = comparison_csv
|
||
path = write_csv(tmp_path / "duplicate.csv", fields, [*rows[:-1], rows[-2]])
|
||
program = """import sys
|
||
from pathlib import Path
|
||
sys.path.insert(0, sys.argv[1])
|
||
import generate_plots
|
||
try:
|
||
generate_plots.load_results(Path(sys.argv[2]))
|
||
except ValueError as exc:
|
||
if 'Duplicate configuration' not in str(exc): raise
|
||
else:
|
||
raise RuntimeError('Invalid matrix accepted with Python -O')
|
||
"""
|
||
subprocess.run([sys.executable, "-O", "-c", program, str(TOOLS / "pimcomp/compare"), str(path)], check=True)
|
||
|
||
|
||
@pytest.mark.parametrize("field,value", [("pimcomp_latency_ms", "999"),
|
||
("pimcomp_throughput_samples_s", "999"), ("pimcomp_power_mw", "999"),
|
||
("pimcomp_energy_pj", "999"), ("pimcomp_functional_validation", "FAIL")])
|
||
def test_repeated_pimcomp_batch_results_must_agree(comparison, comparison_csv, tmp_path, field, value):
|
||
fields, rows = comparison_csv
|
||
row = next(row for row in rows if row["arch"] == "arch-a" and row["model"] == "vgg8"
|
||
and row["raptor_pipeline"] == "4")
|
||
row[field] = value
|
||
with pytest.raises(ValueError, match="inconsistent shared PIMCOMP batch results"):
|
||
comparison.load_results(write_csv(tmp_path / "results.csv", fields, rows))
|
||
|
||
|
||
@pytest.mark.parametrize("status,expected", [("PASS", ["1.0"]),
|
||
("FAIL", ["1.0", "INVALID"]), ("SKIPPED", ["1.0", "SKIPPED"]),
|
||
("NA", ["1.0", "NA"])])
|
||
@pytest.mark.parametrize("direction", [1, -1])
|
||
def test_value_label_validation_text(comparison, status, expected, direction):
|
||
fig, ax = comparison.plt.subplots()
|
||
try:
|
||
comparison.value_label(ax, 0, 1, "1.0", status, direction)
|
||
assert [text.get_text() for text in ax.texts] == expected
|
||
assert ax.texts[-1].get_color() == (comparison.ERROR_COLOR if status == "FAIL" else comparison.TEXT_COLOR)
|
||
if status != "PASS":
|
||
value, validation = ax.texts
|
||
assert value.xy == validation.xy
|
||
assert value.get_position()[1] == validation.get_position()[1] == 0
|
||
assert value.get_position()[0] == -validation.get_position()[0] != 0
|
||
finally:
|
||
comparison.plt.close(fig)
|
||
|
||
|
||
@pytest.mark.parametrize("values", [(1, 2), (-1, -2), (-1, 2), (0, 0)])
|
||
def test_value_label_limits_are_tight_shared_and_unclipped(comparison, values):
|
||
fig, axes = comparison.plt.subplots(1, 2, sharey=True, figsize=(10, 4.4), layout="constrained")
|
||
try:
|
||
for ax, value in zip(axes, values):
|
||
ax.set_ylim(-10, 10)
|
||
ax.bar(0, value)
|
||
direction = -1 if value < 0 else 1
|
||
comparison.value_label(ax, 0, value + direction * 0.05, f"{value:+.2f}%", "FAIL", direction)
|
||
comparison.fit_value_labels(fig)
|
||
fig.canvas.draw()
|
||
renderer = fig.canvas.get_renderer()
|
||
assert axes[0].get_ylim() == axes[1].get_ylim()
|
||
lower, upper = axes[0].get_ylim()
|
||
assert upper - lower < 20
|
||
assert lower <= min(0, *values) and upper >= max(0, *values)
|
||
assert axes[0].get_window_extent(renderer).height == pytest.approx(axes[1].get_window_extent(renderer).height)
|
||
for ax in axes:
|
||
bounds = ax.get_window_extent(renderer)
|
||
for text in ax.texts:
|
||
extent = text.get_window_extent(renderer)
|
||
assert bounds.y0 <= extent.y0 <= extent.y1 <= bounds.y1
|
||
finally:
|
||
comparison.plt.close(fig)
|
||
|
||
|
||
@pytest.mark.parametrize("statuses,expected", [
|
||
(("PASS", "PASS", "PASS"), "PASS"),
|
||
(("FAIL", "PASS", "PASS"), "FAIL"),
|
||
(("PASS", "FAIL", "PASS"), "FAIL"),
|
||
(("PASS", "PASS", "FAIL"), "FAIL"),
|
||
(("PASS", "SKIPPED", "NA"), "SKIPPED"),
|
||
(("PASS", "PASS", "NA"), "N/A"),
|
||
])
|
||
def test_compact_validation_combines_pipeline_statuses(comparison, tmp_path, monkeypatch, statuses, expected):
|
||
rows = comparison.load_results(TOOLS / "pimcomp/compare/results_comparison.csv")
|
||
stages = dict(zip((2, 4, 8), statuses))
|
||
rows = [replace(row, raptor_validation=stages[row.pipeline])
|
||
if row.arch == "arch-a" and row.model == "vgg8" and row.mode == "throughput"
|
||
else row for row in rows]
|
||
captured = []
|
||
monkeypatch.setattr(comparison, "save_figure", lambda fig, name, path: captured.append((fig, name)))
|
||
comparison.draw_validation(rows, tmp_path, compact=True)
|
||
fig, name = captured[0]
|
||
try:
|
||
assert name == "validation_status_compact"
|
||
assert fig.get_size_inches() == pytest.approx((8.5, 3.3))
|
||
for index, ax in enumerate(fig.axes):
|
||
assert [label.get_text() for label in ax.get_xticklabels(minor=True)] == [
|
||
"Raptor", "PIMCOMP", "Raptor", "PIMCOMP"]
|
||
assert [label.get_text() for label in ax.get_yticklabels()] == ([
|
||
"VGG-8", "ResNet-18", "ResNet-34", "GoogLeNet", "YOLO11n"] if index == 0 else [])
|
||
assert len(ax.patches) == len(ax.texts) == 4 * len(comparison.MODELS)
|
||
assert [label.get_text() for label in ax.get_xticklabels()] == ["Latency-mode", "Throughput-mode"]
|
||
assert all(text.get_fontsize() == 12 and text.get_weight() == "normal" for text in ax.texts)
|
||
assert all(text.get_color() == "#1D2530" for text in ax.texts)
|
||
ax = fig.axes[0]
|
||
throughput_x = ax.xaxis.get_minorticklocs()[2]
|
||
assert [text.get_text() for text in ax.texts if text.get_position() == (throughput_x, 0)] == [expected]
|
||
finally:
|
||
comparison.plt.close(fig)
|
||
|
||
|
||
@pytest.mark.parametrize("rates,stage", [((300, 200, 100), 2), ((100, 300, 200), 4),
|
||
((100, 200, 300), 8), ((math.nan, 200, 300), 8), ((300, 300, 200), 2),
|
||
((math.nan, math.nan, math.nan), 2)])
|
||
def test_compact_speedups_select_best_pipeline(comparison, rates, stage):
|
||
template = comparison.load_results(TOOLS / "pimcomp/compare/results_comparison.csv")[0]
|
||
rows = [replace(template, latency_speedup=0.5)]
|
||
rows += [replace(template, mode="throughput", pipeline=depth, raptor_throughput=rate,
|
||
pimcomp_throughput=100, raptor_validation="FAIL")
|
||
for depth, rate in zip((2, 4, 8), rates)]
|
||
latency, throughput = comparison.select_compact_speedups(rows)
|
||
assert latency is rows[0] and latency.speedup == 0.5
|
||
assert throughput.pipeline == stage and throughput.raptor_validation == "FAIL"
|
||
|
||
|
||
@pytest.mark.parametrize("diverging", [False, True])
|
||
@pytest.mark.parametrize("metric,ylabel", [("speedup", "Speedup (x)"),
|
||
("energy", "Energy reduction (x)"), ("power", "Average power reduction (x)")])
|
||
def test_compact_metric_bars_and_units(comparison, tmp_path, monkeypatch, diverging, metric, ylabel):
|
||
from matplotlib.colors import to_rgba
|
||
|
||
template = comparison.load_results(TOOLS / "pimcomp/compare/results_comparison.csv")[0]
|
||
rows = [replace(template, model=model, arch=arch, mode=mode, pipeline=stage,
|
||
latency_speedup=0.5 if arch == "arch-a" else 0.25,
|
||
energy=(0.5 if arch == "arch-a" else 0.25) if mode == "latency" else
|
||
(3 if arch == "arch-a" else 4),
|
||
power=(0.5 if arch == "arch-a" else 0.25) if mode == "latency" else
|
||
(3 if arch == "arch-a" else 4),
|
||
raptor_throughput=300 if arch == "arch-a" else 400,
|
||
pimcomp_throughput=100, pimcomp_validation="FAIL")
|
||
for arch in reversed(comparison.ARCHITECTURES) for model in ("resnet18", "vgg8")
|
||
for mode, stage in (("throughput", 4), ("latency", 1))]
|
||
captured = []
|
||
monkeypatch.setattr(comparison, "save_figure", lambda fig, name, path: captured.append(fig))
|
||
comparison.draw_metric(rows, metric, ylabel, "compact", tmp_path,
|
||
diverging=diverging, compact=True)
|
||
fig = captured[0]
|
||
try:
|
||
assert fig.get_size_inches() == pytest.approx((8.5, 4.4))
|
||
expected_panels = {
|
||
"speedup": ["Latency-mode", "Throughput-mode"],
|
||
"energy": ["Latency-mode energy", "Throughput-mode energy"],
|
||
"power": ["Latency-mode power", "Throughput-mode power"],
|
||
}
|
||
assert [text.get_text() for text in fig.legends[0].get_texts()] == ["Arch-A", "Arch-B"]
|
||
for index, ax in enumerate(fig.axes):
|
||
bars = [container.patches[0] for container in ax.containers]
|
||
assert len(bars) == 4
|
||
assert ax.get_xlabel() == expected_panels[metric][index]
|
||
assert [text.get_text() for text in ax.get_xticklabels()] == ["VGG-8", "ResNet-18"]
|
||
assert list(ax.get_xticks()) == [0.5, 3.5]
|
||
lower, upper = ax.get_xlim()
|
||
assert bars[0].get_x() - lower == pytest.approx(0.4)
|
||
assert upper - bars[-1].get_x() - bars[-1].get_width() == pytest.approx(0.4)
|
||
for offset in (0, 2):
|
||
arch_a, arch_b = bars[offset:offset + 2]
|
||
assert arch_a.get_x() + arch_a.get_width() == arch_b.get_x()
|
||
for bar, arch in zip((arch_a, arch_b), comparison.ARCHITECTURES):
|
||
fill, border, pattern = comparison.ARCH_STYLES[arch]
|
||
assert bar.get_facecolor() == to_rgba(fill)
|
||
assert bar.get_edgecolor() == to_rgba(border)
|
||
assert bar.get_hatch() == pattern
|
||
assert arch_a.get_height() == ((-1 if diverging else 0.5) if index == 0 else (2 if diverging else 3))
|
||
assert arch_b.get_height() == ((-3 if diverging else 0.25) if index == 0 else (3 if diverging else 4))
|
||
assert bars[1].get_x() + bars[1].get_width() < bars[2].get_x()
|
||
assert ax.get_ylabel() == (ylabel if index == 0 else "")
|
||
assert bool(ax.get_yticklabels()) == (index == 0)
|
||
assert all("×" not in label.get_text() for label in ax.get_yticklabels())
|
||
values = (["2.00×", "4.00×"] if diverging else ["0.50×", "0.25×"]) if index == 0 else ["3.00×", "4.00×"]
|
||
assert [text.get_text() for text in ax.texts] == [value for _ in range(2) for number in values
|
||
for value in (number, "P. INVALID")]
|
||
finally:
|
||
comparison.plt.close(fig)
|
||
|
||
|
||
@pytest.mark.parametrize("minimum,expected", [(0.4, 0.25), (0.5, 0.5), (0.8, 0.5)])
|
||
@pytest.mark.parametrize("compact", [False, True])
|
||
def test_ratio_bar_minimum_keeps_values_and_labels_visible(comparison, tmp_path, monkeypatch,
|
||
minimum, expected, compact):
|
||
template = comparison.load_results(TOOLS / "pimcomp/compare/results_comparison.csv")[0]
|
||
rows = [replace(template, arch=arch, mode=mode, pipeline=stage,
|
||
power=minimum if arch == "arch-a" else 2 if mode == "latency" else math.nan)
|
||
for arch in comparison.ARCHITECTURES for mode, stage in (("latency", 1), ("throughput", 4))]
|
||
captured = []
|
||
monkeypatch.setattr(comparison, "save_figure", lambda fig, *args: captured.append(fig))
|
||
comparison.draw_metric(rows, "power", "Average power reduction (x)", "power", tmp_path, compact=compact)
|
||
fig = captured[0]
|
||
try:
|
||
fig.canvas.draw()
|
||
renderer = fig.canvas.get_renderer()
|
||
assert fig.axes[0].get_ylim() == fig.axes[1].get_ylim()
|
||
for index, ax in enumerate(fig.axes):
|
||
assert ax.get_ylim()[0] == expected
|
||
if index == 0:
|
||
assert f"{expected:g}" in [text.get_text() for text in ax.get_yticklabels(which="both")]
|
||
else:
|
||
assert not ax.get_yticklabels(which="both")
|
||
bounds = ax.get_window_extent(renderer)
|
||
for text in ax.texts:
|
||
extent = text.get_window_extent(renderer)
|
||
assert bounds.y0 <= extent.y0 <= extent.y1 <= bounds.y1
|
||
assert any(text.get_text() == "N/A" for ax in fig.axes for text in ax.texts)
|
||
finally:
|
||
comparison.plt.close(fig)
|
||
|
||
|
||
def test_compact_summary_uses_lower_minimum_when_needed(comparison, tmp_path, monkeypatch):
|
||
rows = comparison.load_results(TOOLS / "pimcomp/compare/results_comparison.csv")
|
||
rows = [replace(row, energy=0.4) if row.model == "vgg8" and row.mode == "latency" else row for row in rows]
|
||
captured = []
|
||
monkeypatch.setattr(comparison, "save_figure", lambda fig, *args: captured.append(fig))
|
||
comparison.draw_compact_summary(rows, tmp_path)
|
||
fig = captured[0]
|
||
try:
|
||
fig.canvas.draw()
|
||
assert all(ax.get_xlim()[0] == 0.25 for ax in fig.axes)
|
||
assert "0.25" in [text.get_text() for text in fig.axes[0].get_xticklabels()]
|
||
finally:
|
||
comparison.plt.close(fig)
|
||
|
||
|
||
@pytest.mark.parametrize("missing_energy", [False, True])
|
||
def test_compact_metrics_use_speedup_pipeline_selection(comparison, monkeypatch, missing_energy):
|
||
rows = comparison.load_results(TOOLS / "pimcomp/compare/results_comparison.csv")
|
||
cases = {2: (100, 9, 8), 4: (300, math.nan if missing_energy else 0.5, 0.6), 8: (200, 7, 6)}
|
||
rows = [replace(row, raptor_throughput=cases[row.pipeline][0],
|
||
energy=cases[row.pipeline][1], power=cases[row.pipeline][2])
|
||
if row.arch == "arch-a" and row.model == "vgg8" and row.mode == "throughput"
|
||
else row for row in rows]
|
||
captured = []
|
||
|
||
def capture(selected, field, ylabel, basename, output_dir, diverging=False, compact=False):
|
||
if compact:
|
||
captured.append((selected, field, basename, diverging))
|
||
|
||
monkeypatch.setattr(comparison, "load_results", lambda path: rows)
|
||
for name in ("configure_style", "draw_validation", "draw_pipeline_effect", "draw_absolute", "draw_compact_summary"):
|
||
monkeypatch.setattr(comparison, name, lambda *args, **kwargs: None)
|
||
monkeypatch.setattr(comparison, "draw_metric", capture)
|
||
monkeypatch.setattr(sys, "argv", ["generate_plots.py"])
|
||
comparison.main()
|
||
assert {(field, basename, diverging) for _, field, basename, diverging in captured} == {
|
||
(metric, f"{metric}_compact_{suffix}", diverging)
|
||
for metric in ("speedup", "energy", "power")
|
||
for suffix, diverging in (("comparison", False), ("diverging", True))}
|
||
assert len(captured) == 6
|
||
selected = captured[0][0]
|
||
assert all(plotted is selected for plotted, _, _, _ in captured)
|
||
throughput = next(row for row in selected if row.arch == "arch-a" and row.model == "vgg8"
|
||
and row.mode == "throughput")
|
||
assert throughput.pipeline == 4 and throughput.power == 0.6
|
||
assert math.isnan(throughput.energy) if missing_energy else throughput.energy == 0.5
|
||
|
||
|
||
def test_compiler_palette_and_grayscale_patterns(comparison, tmp_path, monkeypatch):
|
||
from matplotlib.colors import to_rgba
|
||
|
||
assert comparison.COMPILER_STYLES == {
|
||
"raptor": ("#B3D5CA", "#4B7C6A"), "pimcomp": ("#E1BC8B", "#856A44")}
|
||
assert comparison.ARCH_STYLES == {
|
||
"arch-a": ("#B3CCE3", "#456B90", ""), "arch-b": ("#B3D5CA", "#4B7C6A", "//")}
|
||
assert comparison.VALIDATION_STYLES["PASS"] == ("#B3D5CA", "#4B7C6A")
|
||
assert comparison.TEXT_COLOR == "#222222"
|
||
assert comparison.NETWORK_STYLES["yolo11n"] == ("#FF9494", "#C52020")
|
||
assert comparison.NETWORK_STYLES["googlenet"] == ("#D1C0E0", "#77558F")
|
||
rows = comparison.load_results(TOOLS / "pimcomp/compare/results_comparison.csv")
|
||
selected = [replace(row, power=2 if row.arch == "arch-a" else 0.5)
|
||
for row in rows if row.model == "vgg8" and row.mode == "latency"]
|
||
captured = []
|
||
monkeypatch.setattr(comparison, "save_figure", lambda fig, *args: captured.append(fig))
|
||
comparison.draw_metric(selected, "power", "Average power reduction (x)", "power", tmp_path, diverging=True)
|
||
comparison.draw_absolute(selected, "latency", "Single-inference latency (ms)", "latency", tmp_path)
|
||
try:
|
||
for ax, compiler in zip(captured[0].axes, ("raptor", "pimcomp")):
|
||
bar = ax.containers[0].patches[0]
|
||
assert bar.get_facecolor() == to_rgba(comparison.COMPILER_STYLES[compiler][0])
|
||
assert bar.get_edgecolor() == to_rgba(comparison.COMPILER_STYLES[compiler][1])
|
||
assert not bar.get_hatch()
|
||
for ax in captured[1].axes:
|
||
assert [container.patches[0].get_hatch() for container in ax.containers] == ["", ""]
|
||
finally:
|
||
for fig in captured:
|
||
comparison.plt.close(fig)
|
||
|
||
|
||
@pytest.mark.parametrize("hatch", ["//", "\\\\", ".."])
|
||
@pytest.mark.parametrize("width,height", [(20, 130), (160, 15), (20, 1)])
|
||
@pytest.mark.parametrize("clipped", [False, True])
|
||
def test_hatch_geometry_stays_inside_bar(comparison, monkeypatch, hatch, width, height, clipped):
|
||
from matplotlib.backends.backend_agg import RendererAgg
|
||
from matplotlib.path import Path as PlotPath
|
||
from matplotlib.transforms import Affine2D, Bbox
|
||
from plot_style import CenteredHatch
|
||
|
||
renderer = RendererAgg(400, 300, 100)
|
||
gc = renderer.new_gc()
|
||
gc.set_hatch(hatch)
|
||
gc.set_hatch_color((0, 0, 0, 1))
|
||
path = PlotPath.unit_rectangle()
|
||
transform = Affine2D().scale(width, height).translate(50, 70)
|
||
bar = transform.transform_path(path).get_extents()
|
||
if clipped:
|
||
gc.set_clip_rectangle(Bbox.from_extents(55, 70.25, 180, 180))
|
||
bar = Bbox.intersection(bar, gc.get_clip_rectangle())
|
||
drawn = []
|
||
|
||
def capture(context, geometry, affine, face=None):
|
||
drawn.append((context.get_hatch(), affine.transform_path(geometry).get_extents()))
|
||
|
||
def capture_markers(context, marker, marker_transform, centers, affine, face=None):
|
||
for x, y in affine.transform_path(centers).vertices:
|
||
capture(context, marker, marker_transform + Affine2D().translate(x, y), face)
|
||
|
||
monkeypatch.setattr(renderer, "draw_path", capture)
|
||
monkeypatch.setattr(renderer, "draw_markers", capture_markers)
|
||
CenteredHatch().draw_path(renderer, gc, path, transform, (1, 1, 1, 1))
|
||
assert len(drawn) > 1
|
||
assert all(pattern is None for pattern, _ in drawn)
|
||
for _, bounds in drawn[1:]:
|
||
assert bar.x0 - 1e-8 <= bounds.x0 <= bounds.x1 <= bar.x1 + 1e-8
|
||
assert bar.y0 - 1e-8 <= bounds.y0 <= bounds.y1 <= bar.y1 + 1e-8
|
||
|
||
|
||
def test_tick_rotation_changes_only_overlapping_labels(comparison):
|
||
fig, ax = comparison.plt.subplots(figsize=(4, 3), layout="constrained")
|
||
try:
|
||
ax.set_xlim(-1, 6)
|
||
ax.set_xticks([0, 0.5, 5], ["Single-inference", "Throughput", "8"])
|
||
comparison.fit_x_tick_labels(fig)
|
||
assert [text.get_rotation() for text in ax.get_xticklabels()] == [330, 330, 0]
|
||
assert [text.get_ha() for text in ax.get_xticklabels()] == ["left", "left", "center"]
|
||
finally:
|
||
comparison.plt.close(fig)
|
||
|
||
|
||
def test_all_plot_terminology(comparison, monkeypatch):
|
||
from matplotlib.path import Path as PlotPath
|
||
from matplotlib.text import Text
|
||
from matplotlib.transforms import Affine2D
|
||
|
||
plotter = comparison
|
||
labels = set()
|
||
|
||
def capture(fig, name, output):
|
||
fig.canvas.draw()
|
||
if "compact" in name:
|
||
compact_text = "\n".join(text.get_text().lower() for text in fig.findobj(Text))
|
||
assert "single-inference" not in compact_text
|
||
assert "best-case throughput" not in compact_text
|
||
if name == "performance_energy_compact":
|
||
labels.update(text.get_text() for text in fig.findobj(Text))
|
||
plotter.plt.close(fig)
|
||
return
|
||
if name != "raptor_pipeline_throughput":
|
||
assert fig.axes[1].get_ylabel() == ""
|
||
assert fig.axes[1].get_yticklabels() == []
|
||
renderer = fig.canvas.get_renderer()
|
||
for ax in fig.axes:
|
||
for container in ax.containers:
|
||
for bar in container.patches:
|
||
center = bar.get_x() + bar.get_width() / 2
|
||
bounds = bar.get_window_extent(renderer)
|
||
for text in ax.texts:
|
||
if hasattr(text, "xy") and text.xy[0] == center:
|
||
extent = text.get_window_extent(renderer)
|
||
assert bounds.x0 <= extent.x0 <= extent.x1 <= bounds.x1, (name, text.get_text())
|
||
for legend in fig.legends:
|
||
bounds = legend.get_window_extent(renderer)
|
||
assert fig.bbox.x0 <= bounds.x0 <= bounds.x1 <= fig.bbox.x1, name
|
||
tick_paths = []
|
||
for ax in fig.axes:
|
||
for minor in (False, True):
|
||
for text in ax.get_xticklabels(minor=minor):
|
||
if not text.get_visible() or not text.get_text():
|
||
continue
|
||
angle = text.get_rotation()
|
||
assert angle in (0, 330), (name, text.get_text(), angle)
|
||
bounds = text.get_window_extent(renderer)
|
||
text.set_rotation(0)
|
||
unrotated = text.get_window_extent(renderer)
|
||
text.set_rotation(angle)
|
||
path = PlotPath.unit_rectangle().transformed(
|
||
Affine2D().scale(unrotated.width, unrotated.height).rotate_deg(angle))
|
||
rotated = path.get_extents()
|
||
path = path.transformed(Affine2D().translate(bounds.x0 - rotated.x0, bounds.y0 - rotated.y0))
|
||
assert not any(path.intersects_path(other, filled=True) for other in tick_paths), name
|
||
tick_paths.append(path)
|
||
if text.get_text() in comparison.MODEL_NAMES.values():
|
||
assert angle == 0, (name, text.get_text())
|
||
heights = [ax.get_window_extent(renderer).height for ax in fig.axes]
|
||
assert max(heights) - min(heights) < 1, name
|
||
for ax in fig.axes:
|
||
bounds = ax.get_window_extent(renderer)
|
||
for text in ax.texts:
|
||
extent = text.get_window_extent(renderer)
|
||
assert bounds.y0 - 1 <= extent.y0 <= extent.y1 <= bounds.y1 + 1, (name, text.get_text())
|
||
assert fig.bbox.x0 <= extent.x0 <= extent.x1 <= fig.bbox.x1, (name, text.get_text())
|
||
labels.update(text.get_text() for text in fig.findobj(Text))
|
||
plotter.plt.close(fig)
|
||
|
||
monkeypatch.setattr(plotter, "save_figure", capture)
|
||
monkeypatch.setattr(sys, "argv", ["generate_plots.py"])
|
||
plotter.main()
|
||
text = "\n".join(labels)
|
||
for old in ("Pimcomp", "ARCHITECTURE", "VGG8", "RESNET", "GOOGLENET", "YOLO11N",
|
||
"Single inference", "Best case", "Latency, 1 stage", "Energy per inference advantage",
|
||
"Energy-per-inference advantage", "Energy consumption reduction", "power advantage"):
|
||
assert old not in text
|
||
assert {"VGG-8", "ResNet-18", "ResNet-34", "GoogLeNet"} <= labels
|
||
assert "Arch-A" in text and "Arch-B" in text
|
||
assert {"Raptor", "PIMCOMP", "YOLO11n", "Single-inference", "2 stages", "4 stages", "8 stages",
|
||
"Single-inference latency (ms)", "Throughput (inferences/s)", "Energy per inference (mJ)",
|
||
"Energy reduction (x)", "Average power reduction (x)", "Derived as 1 / latency",
|
||
"Throughput measurement"} <= labels
|
||
|
||
|
||
@pytest.mark.parametrize("width,one_row", [(12, True), (4.5, False), (3, False)])
|
||
def test_legend_uses_available_width_and_balances_rows(comparison, width, one_row):
|
||
from collections import Counter
|
||
from matplotlib.patches import Patch
|
||
|
||
fig = comparison.plt.figure(figsize=(width, 4), layout="constrained")
|
||
try:
|
||
handles = [Patch(label=f"Entry {index}") for index in range(7)]
|
||
comparison.add_legend(fig, handles)
|
||
fig.canvas.draw()
|
||
renderer = fig.canvas.get_renderer()
|
||
legend = fig.legends[0]
|
||
counts = Counter(round(text.get_window_extent(renderer).y0, 3)
|
||
for text in legend.get_texts() if text.get_visible())
|
||
assert sum(counts.values()) == len(handles)
|
||
assert (len(counts) == 1) == one_row
|
||
assert max(counts.values()) - min(counts.values()) <= 1
|
||
bounds = legend.get_window_extent(renderer)
|
||
assert 0 <= bounds.x0 <= bounds.x1 <= fig.bbox.width
|
||
finally:
|
||
comparison.plt.close(fig)
|
||
|
||
|
||
@pytest.mark.parametrize("diverging", [False, True])
|
||
def test_advantage_throughput_groups_touch(comparison, tmp_path, monkeypatch, diverging):
|
||
rows = comparison.load_results(TOOLS / "pimcomp/compare/results_comparison.csv")
|
||
selected = [row for row in rows if row.model in ("vgg8", "resnet18")]
|
||
captured = []
|
||
monkeypatch.setattr(comparison, "save_figure", lambda fig, *args: captured.append(fig))
|
||
comparison.draw_metric(selected, "energy", "Energy reduction (x)", "energy", tmp_path,
|
||
diverging=diverging)
|
||
fig = captured[0]
|
||
try:
|
||
for ax in fig.axes:
|
||
bars = [container.patches[0] for container in ax.containers]
|
||
lower, upper = ax.get_xlim()
|
||
assert bars[0].get_x() - lower == pytest.approx(0.4)
|
||
assert upper - bars[-1].get_x() - bars[-1].get_width() == pytest.approx(0.4)
|
||
for index, (left, right) in enumerate(zip(bars, bars[1:])):
|
||
edge = left.get_x() + left.get_width()
|
||
if index % 4 in (1, 2):
|
||
assert edge == pytest.approx(right.get_x())
|
||
else:
|
||
assert edge < right.get_x()
|
||
finally:
|
||
comparison.plt.close(fig)
|
||
|
||
|
||
def test_grouped_absolute_plot_scheduling_labels(comparison, tmp_path, monkeypatch):
|
||
rows = comparison.load_results(TOOLS / "pimcomp/compare/results_comparison.csv")
|
||
captured = []
|
||
monkeypatch.setattr(comparison, "save_figure", lambda fig, *args: captured.append(fig))
|
||
comparison.draw_absolute(rows, "energy", "Energy per inference (mJ)", "energy", tmp_path, scale=1e-9)
|
||
fig = captured[0]
|
||
try:
|
||
for ax in fig.axes:
|
||
assert {text.get_text() for text in ax.get_xticklabels(minor=True)} == {"Latency", "Throughput"}
|
||
labels = [text.get_text() for text in fig.legends[0].get_texts()]
|
||
assert "1 stage (latency)" in labels and "Single-inference" not in labels
|
||
finally:
|
||
comparison.plt.close(fig)
|
||
|
||
|
||
@pytest.mark.parametrize("status", ["PASS", "FAIL", "SKIPPED", "NA"])
|
||
def test_pipeline_plot_connects_derived_and_measured_rates(comparison, tmp_path, monkeypatch, status):
|
||
rows = comparison.load_results(TOOLS / "pimcomp/compare/results_comparison.csv")
|
||
rows = [replace(row, raptor_validation=status, pimcomp_validation=status) for row in rows]
|
||
captured = []
|
||
monkeypatch.setattr(comparison, "save_figure", lambda fig, *args: captured.append(fig))
|
||
comparison.draw_pipeline_effect(rows, tmp_path)
|
||
fig = captured[0]
|
||
try:
|
||
assert fig.get_size_inches() == pytest.approx((13, 4.3))
|
||
assert fig._supylabel.get_text() == "Throughput (inferences/s)"
|
||
for index, (ax, (arch, model)) in enumerate(zip(fig.axes, ((arch, model) for arch in comparison.ARCHITECTURES
|
||
for model in comparison.MODELS))):
|
||
selected = [row for row in rows if row.arch == arch and row.model == model]
|
||
measured_line, derived_marker, *measured_markers = ax.lines
|
||
assert measured_line.get_ydata()[0] == pytest.approx(1000 / selected[0].raptor_latency)
|
||
assert list(measured_line.get_ydata()[1:]) == [row.raptor_throughput for row in selected[1:]]
|
||
assert derived_marker.get_marker() == "s"
|
||
assert derived_marker.get_ydata()[0] == pytest.approx(1000 / selected[0].raptor_latency)
|
||
assert derived_marker.get_color() == comparison.NETWORK_STYLES[model][1]
|
||
assert all(marker.get_marker() == "o" for marker in measured_markers)
|
||
assert measured_line.get_color() == comparison.NETWORK_STYLES[model][1]
|
||
assert [label.get_text() for label in ax.get_xticklabels()] == (
|
||
["1", "2", "4", "8"] if arch == "arch-b" else [])
|
||
assert ax.get_xlabel() == ""
|
||
assert ax.get_title() == (comparison.MODEL_NAMES[model] if arch == "arch-a" else "")
|
||
assert ax.title.get_weight() == "normal"
|
||
if model == comparison.MODELS[-1]:
|
||
assert ax.get_ylabel() == comparison.ARCH_NAMES[arch]
|
||
assert ax.yaxis.get_label_position() == "right"
|
||
assert ax.yaxis.label.get_rotation() == 270
|
||
assert ax.yaxis.label.get_weight() == "bold"
|
||
else:
|
||
assert ax.get_ylabel() == ""
|
||
assert all(label.get_rotation() == 0 for label in ax.get_xticklabels())
|
||
lower, upper = ax.get_ylim()
|
||
values = measured_line.get_ydata()
|
||
assert 0 < lower < min(values) <= max(values) < upper
|
||
assert ax.get_yticklabels()
|
||
assert ax.get_ylim() == fig.axes[index % len(comparison.MODELS)].get_ylim()
|
||
assert [text.get_text() for text in fig.legends[0].get_texts()] == [
|
||
"Derived as 1 / latency", "Throughput measurement"]
|
||
assert len({ax.lines[0].get_color() for ax in fig.axes[:len(comparison.MODELS)]}) == len(comparison.MODELS)
|
||
finally:
|
||
comparison.plt.close(fig)
|
||
|
||
|
||
def test_plotter_exports_png_svg_and_pdf(comparison, tmp_path):
|
||
comparison.configure_style()
|
||
rows = comparison.load_results(TOOLS / "pimcomp/compare/results_comparison.csv")
|
||
selected = [replace(row, raptor_validation="FAIL", pimcomp_latency=math.nan)
|
||
for row in rows if row.model == "vgg8" and row.mode == "latency"]
|
||
comparison.draw_absolute(selected, "latency", "Latency (ms)", "comparison-smoke", tmp_path)
|
||
comparison.draw_validation(rows, tmp_path, compact=True)
|
||
comparison.draw_compact_summary(rows, tmp_path)
|
||
compact_names = []
|
||
for metric in ("speedup", "energy", "power"):
|
||
for suffix, diverging in (("comparison", False), ("diverging", True)):
|
||
name = f"{metric}_compact_{suffix}"
|
||
compact_names.append(name)
|
||
comparison.draw_metric(comparison.select_compact_speedups(rows), metric, "Advantage (x)",
|
||
name, tmp_path, diverging=diverging, compact=True)
|
||
for name in ("comparison-smoke", "validation_status_compact", "performance_energy_compact", *compact_names):
|
||
assert (tmp_path / f"png/{name}.png").read_bytes().startswith(b"\x89PNG\r\n\x1a\n")
|
||
svg = (tmp_path / f"svg/{name}.svg").read_text()
|
||
assert "<svg" in svg
|
||
assert (tmp_path / f"pdf/{name}.pdf").read_bytes().startswith(b"%PDF-")
|
||
assert "N/A" in (tmp_path / "svg/comparison-smoke.svg").read_text()
|
||
assert "P. INVALID" in (tmp_path / "svg/comparison-smoke.svg").read_text()
|
||
|
||
|
||
def test_compact_summary_ratios_validity_and_layout(comparison, tmp_path, monkeypatch):
|
||
from matplotlib.path import Path as PlotPath
|
||
from matplotlib.transforms import Affine2D, Bbox
|
||
|
||
rows = comparison.load_results(TOOLS / "pimcomp/compare/results_comparison.csv")
|
||
selected = comparison.select_compact_speedups([row for row in rows if row.model != "yolo11n"])
|
||
lookup = {(row.model, row.arch, row.mode): row for row in selected}
|
||
captured = []
|
||
monkeypatch.setattr(comparison, "save_figure", lambda fig, name, path: captured.append((fig, name)))
|
||
comparison.draw_compact_summary(rows, tmp_path)
|
||
fig, name = captured[0]
|
||
try:
|
||
assert name == "performance_energy_compact"
|
||
assert len(fig.axes) == 4
|
||
fig.canvas.draw()
|
||
renderer = fig.canvas.get_renderer()
|
||
keys = [(model, arch) for model in comparison.MODELS[:-1] for arch in comparison.ARCHITECTURES]
|
||
for ax, mode, field in zip(fig.axes, ("latency", "throughput", "latency", "throughput"),
|
||
("latency_speedup", "throughput_speedup", "energy", "energy")):
|
||
assert ax.get_xscale() == "linear"
|
||
low, high = ax.get_xlim()
|
||
assert low == 0.5 < 1 < high
|
||
points = [line for line in ax.lines if line.get_marker() in ("o", "s", "^", "D")]
|
||
point_keys = [(*key, mode) for key in keys]
|
||
assert len(points) == len(point_keys)
|
||
expected, invalid_flags = [], []
|
||
for point, key in zip(points, point_keys):
|
||
row = lookup[key]
|
||
ratio = getattr(row, field)
|
||
expected.append(ratio)
|
||
assert list(point.get_xdata()) == pytest.approx([ratio])
|
||
assert list(point.get_ydata()) == [0 if row.arch == "arch-a" else 2]
|
||
assert point.get_marker() == {"vgg8": "o", "resnet18": "s", "resnet34": "^", "googlenet": "D"}[row.model]
|
||
fill, border = comparison.NETWORK_STYLES[row.model]
|
||
assert point.get_markerfacecolor() == fill
|
||
assert point.get_markeredgecolor() == border
|
||
invalid = row.mode == "throughput" or row.pimcomp_validation == "FAIL"
|
||
invalid_flags.append(invalid)
|
||
labels = [text for text in ax.texts if getattr(text, "xycoords", None) == "data"]
|
||
groups = []
|
||
for text, key, ratio, invalid in zip(labels, point_keys, expected, invalid_flags):
|
||
assert text.get_text() == f"{ratio:.2f}×"
|
||
assert text.get_math_fontfamily() == "cm"
|
||
assert text.get_color() == comparison.TEXT_COLOR
|
||
assert text.get_rotation() == 90
|
||
assert text.arrow_patch is None
|
||
assert abs(text.get_position()[0]) <= 8
|
||
assert (text.get_position()[1] > 0) == (key[1] == "arch-a")
|
||
children = [child for child in ax.texts if getattr(child, "xycoords", None) is text]
|
||
assert len(children) == 1
|
||
for dagger in children:
|
||
assert dagger.get_text() == r"$\boldsymbol{\dagger}$"
|
||
assert dagger.get_color() == comparison.DAGGER_COLOR == "#AA1C1C"
|
||
assert dagger.get_fontsize() == 9
|
||
assert dagger.get_fontsize() > text.get_fontsize()
|
||
assert dagger.get_alpha() == int(invalid)
|
||
groups.append([text, *children])
|
||
for lane in (0, 2):
|
||
offsets = [text.get_window_extent(renderer).y1 - ax.transData.transform(text.xy)[1]
|
||
for text in labels if text.xy[1] == lane]
|
||
assert max(offsets) == pytest.approx(min(offsets))
|
||
assert len(labels) == 8
|
||
assert list(ax.get_yticks()) == [0, 2]
|
||
assert [text.get_text() for text in ax.get_yticklabels()] == ["Arch-A", "Arch-B"]
|
||
assert ax.get_title() == ""
|
||
assert ax.yaxis.label.get_rotation() == 90
|
||
assert ax.yaxis.label._get_multialignment() == "center"
|
||
assert ax.yaxis.label.get_fontsize() == 8.5
|
||
assert len(ax.get_ylabel().splitlines()) <= 2
|
||
assert ax.yaxis.label.get_window_extent(renderer).x1 < ax.get_window_extent(renderer).x0
|
||
assert bool(ax.get_xticklabels()) == (ax is fig.axes[0])
|
||
if ax is fig.axes[0]:
|
||
assert "0.5" in [text.get_text() for text in ax.get_xticklabels()]
|
||
assert "0" not in [text.get_text() for text in ax.get_xticklabels()]
|
||
assert all(tick.label2.get_visible() and not tick.label1.get_visible()
|
||
for tick in ax.xaxis.get_major_ticks())
|
||
assert not any(text.get_text() in ("Raptor / PIMCOMP", "PIMCOMP / Raptor") for text in ax.texts)
|
||
baseline = next(line for line in ax.lines if list(line.get_xdata()) == [1, 1])
|
||
assert baseline.get_linestyle() == "--"
|
||
bounds = ax.get_window_extent(renderer)
|
||
paths = []
|
||
for line in ax.lines:
|
||
box = line.get_window_extent(renderer).padded(1)
|
||
paths.append(PlotPath.unit_rectangle().transformed(
|
||
Affine2D().scale(box.width, box.height).translate(box.x0, box.y0)))
|
||
for group in groups:
|
||
text = group[0]
|
||
box = Bbox.union([part.get_window_extent(renderer) for part in group])
|
||
assert bounds.x0 <= box.x0 <= box.x1 <= bounds.x1
|
||
assert bounds.y0 <= box.y0 <= box.y1 <= bounds.y1
|
||
path = PlotPath.unit_rectangle().transformed(
|
||
Affine2D().scale(box.width, box.height).translate(box.x0, box.y0))
|
||
assert not any(path.intersects_path(other, filled=True) for other in paths), text.get_text()
|
||
paths.append(path)
|
||
assert "YOLO11n" not in [text.get_text() for text in ax.get_yticklabels()]
|
||
assert {text.get_position()[1] > 0 for ax in fig.axes for text in ax.texts
|
||
if getattr(text, "xycoords", None) == "data"} == {False, True}
|
||
assert all(ax.get_xlim() == fig.axes[0].get_xlim() for ax in fig.axes)
|
||
assert [ax.get_ylabel() for ax in fig.axes] == [
|
||
"Latency-mode\nspeedup", "Throughput-mode\nspeedup",
|
||
"Latency-mode\nenergy reduction", "Throughput-mode\nenergy reduction"]
|
||
assert fig.axes[0].xaxis.get_label_position() == "top"
|
||
xlabel = fig.axes[0].xaxis.label.get_window_extent(renderer)
|
||
legend = fig.legends[0].get_window_extent(renderer)
|
||
assert fig.axes[0].get_window_extent(renderer).y1 < xlabel.y0 < xlabel.y1 < legend.y0
|
||
tick_top = max(text.get_window_extent(renderer).y1 for text in fig.axes[0].get_xticklabels())
|
||
assert 0 < xlabel.y0 - tick_top < renderer.points_to_pixels(12)
|
||
axis_bounds = fig.axes[0].get_window_extent(renderer)
|
||
assert (xlabel.x0 + xlabel.x1) / 2 == pytest.approx((axis_bounds.x0 + axis_bounds.x1) / 2)
|
||
assert fig.axes[0].get_xlabel() == "Ratio (x)"
|
||
assert {"← PIMCOMP better", "Raptor better →"} <= {text.get_text() for text in fig.axes[0].texts}
|
||
left, right = [text.get_window_extent(renderer) for text in fig.axes[0].texts if not hasattr(text, "xy")]
|
||
assert left.y0 == right.y0 == pytest.approx(xlabel.y0)
|
||
assert left.x1 < xlabel.x0 < xlabel.x1 < right.x0
|
||
legend_labels = {text.get_text() for text in fig.legends[0].get_texts() if text.get_visible()}
|
||
assert legend_labels == {"VGG-8", "ResNet-18", "ResNet-34", "GoogLeNet", "PIMCOMP invalid"}
|
||
for handle, text in zip(fig.legends[0].legend_handles, fig.legends[0].get_texts()):
|
||
if text.get_text() in comparison.MODEL_NAMES.values():
|
||
model = next(model for model, name in comparison.MODEL_NAMES.items() if name == text.get_text())
|
||
assert handle.get_marker() == {"vgg8": "o", "resnet18": "s", "resnet34": "^", "googlenet": "D"}[model]
|
||
assert handle.get_markerfacecolor() == comparison.NETWORK_STYLES[model][0]
|
||
elif text.get_text() == "PIMCOMP invalid":
|
||
assert handle.get_text() == r"$\boldsymbol{\dagger}$"
|
||
assert handle.get_fontsize() == 9
|
||
assert handle.get_math_fontfamily() == "cm"
|
||
assert handle.get_color() == comparison.DAGGER_COLOR
|
||
result_dagger = next(child for child in fig.axes[0].texts
|
||
if getattr(getattr(child, "xycoords", None), "xycoords", None) == "data"
|
||
and child.get_alpha() == 1)
|
||
result_box = result_dagger.get_window_extent(renderer)
|
||
legend_box = handle.get_window_extent(renderer)
|
||
assert legend_box.width == pytest.approx(result_box.height)
|
||
assert legend_box.height == pytest.approx(result_box.width)
|
||
ratios = [line.get_xdata()[0] for ax in fig.axes for line in ax.lines
|
||
if line.get_marker() in ("o", "s", "^", "D")]
|
||
lower, upper = fig.axes[0].get_xlim()
|
||
padding = (max(ratios) - min(ratios)) * 0.04
|
||
assert lower == 0.5
|
||
assert upper - max(ratios) == pytest.approx(padding)
|
||
finally:
|
||
comparison.plt.close(fig)
|
||
|
||
|
||
def test_compact_summary_rejects_missing_ratio(comparison, tmp_path):
|
||
rows = comparison.load_results(TOOLS / "pimcomp/compare/results_comparison.csv")
|
||
rows = [replace(row, energy=math.nan) if row.mode == "throughput" and row.model == "vgg8"
|
||
else row for row in rows]
|
||
with pytest.raises(ValueError, match="requires a positive energy ratio"):
|
||
comparison.draw_compact_summary(rows, tmp_path)
|
||
|
||
|
||
@pytest.mark.parametrize("compiler,status,label", [("pimcomp", "FAIL", r"$\boldsymbol{\dagger}$"), ("raptor", "FAIL", "R. INVALID"),
|
||
("pimcomp", "SKIPPED", "NOT VALIDATED"), ("pimcomp", "NA", "NOT VALIDATED")])
|
||
def test_compact_summary_validation_labels(comparison, tmp_path, monkeypatch, compiler, status, label):
|
||
rows = comparison.load_results(TOOLS / "pimcomp/compare/results_comparison.csv")
|
||
rows = [replace(row, **{f"{compiler}_validation": status})
|
||
if row.model == "vgg8" and row.arch == "arch-a" and row.mode == "latency" else row for row in rows]
|
||
captured = []
|
||
monkeypatch.setattr(comparison, "save_figure", lambda fig, *args: captured.append(fig))
|
||
comparison.draw_compact_summary(rows, tmp_path)
|
||
fig = captured[0]
|
||
try:
|
||
ax = fig.axes[0]
|
||
if compiler == "pimcomp" and status == "FAIL":
|
||
assert any(label in text.get_text() and text.get_color() == comparison.DAGGER_COLOR for text in ax.texts)
|
||
else:
|
||
assert label in ax.texts[0].get_text()
|
||
assert ax.texts[0].get_color() == comparison.TEXT_COLOR
|
||
finally:
|
||
comparison.plt.close(fig)
|