from __future__ import annotations import csv import importlib.util import math from pathlib import Path import subprocess import sys import pytest TOOLS = Path(__file__).resolve().parents[1] / "tools/pim" @pytest.fixture def plotters(monkeypatch): modules = [] for name, directory in (("comparison_plots", "pimcomp/compare"), ("ablation_plots", "ablation")): monkeypatch.syspath_prepend(str(TOOLS / directory)) spec = importlib.util.spec_from_file_location(name, TOOLS / directory / "generate_plots.py") module = importlib.util.module_from_spec(spec) monkeypatch.setitem(sys.modules, name, module) spec.loader.exec_module(module) modules.append(module) return modules @pytest.fixture def comparison_csv(): with (TOOLS / "pimcomp/compare/results_comparison.csv").open(newline="") as stream: reader = csv.DictReader(stream) return reader.fieldnames, list(reader) def write_csv(path, fields, rows): with path.open("w", newline="") as stream: writer = csv.DictWriter(stream, fieldnames=fields) writer.writeheader() writer.writerows(rows) return path @pytest.mark.parametrize("custom", [False, True]) def test_comparison_plot_cli_input_paths(plotters, tmp_path, monkeypatch, custom): comparison, _ = plotters expected = tmp_path / "custom.csv" if custom else TOOLS / "pimcomp/compare/results_comparison.csv" seen = [] class InputCaptured(Exception): pass def capture(path): seen.append(path) raise InputCaptured monkeypatch.setattr(comparison, "load_results", capture) monkeypatch.setattr(comparison, "configure_style", lambda: None) monkeypatch.setattr(sys, "argv", ["generate_plots.py", *(["--input", str(expected)] if custom else [])]) with pytest.raises(InputCaptured): comparison.main() assert seen == [expected] def test_exact_comparison_configuration_matrix(plotters, comparison_csv, tmp_path): comparison, _ = plotters fields, rows = comparison_csv results = comparison.load_results(write_csv(tmp_path / "results.csv", fields, rows)) keys = {(row.arch, row.model, row.mode, row.pipeline) for row in results} assert keys == {(arch, model, mode, stage) for arch in comparison.ARCHITECTURES for model in comparison.MODELS for mode, stage in comparison.CONFIGURATIONS} @pytest.mark.parametrize("damage,message", [("missing", "Missing"), ("duplicate", "Duplicate"), ("arch", "Unexpected"), ("model", "Unexpected"), ("pipeline", "Unexpected"), ("mode", "Unexpected")]) def test_comparison_rejects_bad_matrix_keys(plotters, comparison_csv, tmp_path, damage, message): comparison, _ = plotters fields, rows = comparison_csv if damage == "missing": rows.pop() elif damage == "duplicate": rows[-1] = dict(rows[-2]) # Still 40 rows and 20 per architecture. else: field = "raptor_pipeline" if damage == "pipeline" else damage rows[-1][field] = {"arch": "arch-c", "model": "other", "pipeline": "3", "mode": "other"}[damage] with pytest.raises(ValueError, match=message): comparison.load_results(write_csv(tmp_path / "results.csv", fields, rows)) def test_matrix_validation_survives_optimized_python(comparison_csv, tmp_path): fields, rows = comparison_csv path = write_csv(tmp_path / "duplicate.csv", fields, [*rows[:-1], rows[-2]]) program = """import sys from pathlib import Path sys.path.insert(0, sys.argv[1]) import generate_plots try: generate_plots.load_results(Path(sys.argv[2])) except ValueError as exc: if 'Duplicate configuration' not in str(exc): raise else: raise RuntimeError('Invalid matrix accepted with Python -O') """ subprocess.run([sys.executable, "-O", "-c", program, str(TOOLS / "pimcomp/compare"), str(path)], check=True) @pytest.mark.parametrize("field,value", [("pimcomp_latency_ms", "999"), ("pimcomp_throughput_samples_s", "999"), ("pimcomp_power_mw", "999"), ("pimcomp_energy_pj", "999"), ("pimcomp_functional_validation", "FAIL")]) def test_repeated_pimcomp_batch_results_must_agree(plotters, comparison_csv, tmp_path, field, value): comparison, _ = plotters fields, rows = comparison_csv row = next(row for row in rows if row["arch"] == "arch-a" and row["model"] == "vgg8" and row["raptor_pipeline"] == "4") row[field] = value with pytest.raises(ValueError, match="inconsistent shared Pimcomp batch results"): comparison.load_results(write_csv(tmp_path / "results.csv", fields, rows)) def test_legacy_invalid_ablation_pairs_are_unavailable_without_changing_pass(plotters): _, ablation = plotters with pytest.warns(UserWarning, match="2 invalid reciprocal"): rows = ablation.load_results(TOOLS / "ablation/results_ablation.csv") invalid = [row for row in rows if row.model == "googlenet" and row.variant == "no-plan" and row.mode == "throughput"] assert len(invalid) == 2 assert all(row.validation == "PASS" and math.isnan(row.metrics["latency"]) and math.isnan(row.metrics["throughput"]) for row in invalid) def test_ablation_missing_values_and_legitimate_extremes(plotters, tmp_path): _, ablation = plotters row = {"variant": "no-plan", "arch": "arch-a", "model": "vgg8", "mode": "throughput", "raptor_pipeline": "4", "functional_validation": "FAIL", "latency": "-100.00%", "throughput": "+100000000.00%", "power": "NA", "energy": "-100.00%"} result = ablation.load_results(write_csv(tmp_path / "extreme.csv", list(row), [row]))[0] assert result.validation == "FAIL" assert result.metrics["latency"] == result.metrics["energy"] == -100 assert result.metrics["throughput"] == 100000000 assert math.isnan(result.metrics["power"]) def test_both_plotters_export_png_and_svg(plotters, tmp_path): comparison, ablation = plotters comparison.configure_style() rows = comparison.load_results(TOOLS / "pimcomp/compare/results_comparison.csv") selected = [row for row in rows if row.model == "vgg8" and row.mode == "latency"] comparison.draw_absolute(selected, "latency", "Latency (ms)", "comparison-smoke", tmp_path) row = ablation.Result("baseline", "arch-a", "vgg8", "latency", 1, "PASS", {"latency": math.nan, "power": 0, "energy": 0, "throughput": math.nan}) ablation.draw_metric([row], "latency", "Latency change (%)", tmp_path) for name in ("comparison-smoke", "latency_pipeline1_latency"): assert (tmp_path / f"png/{name}.png").read_bytes().startswith(b"\x89PNG\r\n\x1a\n") svg = (tmp_path / f"svg/{name}.svg").read_text() assert "