NiccoloN
034c197260
Validate Operations / validate-operations (push) Failing after 3h10m55s
add plot generation scripts adjust validation tools add up-to-date results
158 lines
6.8 KiB
Python
158 lines
6.8 KiB
Python
from __future__ import annotations
|
|
|
|
import csv
|
|
import importlib.util
|
|
import math
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
|
|
import pytest
|
|
|
|
|
|
TOOLS = Path(__file__).resolve().parents[1] / "tools/pim"
|
|
|
|
|
|
@pytest.fixture
|
|
def plotters(monkeypatch):
|
|
modules = []
|
|
for name, directory in (("comparison_plots", "pimcomp/compare"), ("ablation_plots", "ablation")):
|
|
monkeypatch.syspath_prepend(str(TOOLS / directory))
|
|
spec = importlib.util.spec_from_file_location(name, TOOLS / directory / "generate_plots.py")
|
|
module = importlib.util.module_from_spec(spec)
|
|
monkeypatch.setitem(sys.modules, name, module)
|
|
spec.loader.exec_module(module)
|
|
modules.append(module)
|
|
return modules
|
|
|
|
|
|
@pytest.fixture
|
|
def comparison_csv():
|
|
with (TOOLS / "pimcomp/compare/results_comparison.csv").open(newline="") as stream:
|
|
reader = csv.DictReader(stream)
|
|
return reader.fieldnames, list(reader)
|
|
|
|
|
|
def write_csv(path, fields, rows):
|
|
with path.open("w", newline="") as stream:
|
|
writer = csv.DictWriter(stream, fieldnames=fields)
|
|
writer.writeheader()
|
|
writer.writerows(rows)
|
|
return path
|
|
|
|
|
|
@pytest.mark.parametrize("custom", [False, True])
|
|
def test_comparison_plot_cli_input_paths(plotters, tmp_path, monkeypatch, custom):
|
|
comparison, _ = plotters
|
|
expected = tmp_path / "custom.csv" if custom else TOOLS / "pimcomp/compare/results_comparison.csv"
|
|
seen = []
|
|
|
|
class InputCaptured(Exception):
|
|
pass
|
|
|
|
def capture(path):
|
|
seen.append(path)
|
|
raise InputCaptured
|
|
|
|
monkeypatch.setattr(comparison, "load_results", capture)
|
|
monkeypatch.setattr(comparison, "configure_style", lambda: None)
|
|
monkeypatch.setattr(sys, "argv", ["generate_plots.py", *(["--input", str(expected)] if custom else [])])
|
|
with pytest.raises(InputCaptured):
|
|
comparison.main()
|
|
assert seen == [expected]
|
|
|
|
|
|
def test_exact_comparison_configuration_matrix(plotters, comparison_csv, tmp_path):
|
|
comparison, _ = plotters
|
|
fields, rows = comparison_csv
|
|
results = comparison.load_results(write_csv(tmp_path / "results.csv", fields, rows))
|
|
keys = {(row.arch, row.model, row.mode, row.pipeline) for row in results}
|
|
assert keys == {(arch, model, mode, stage) for arch in comparison.ARCHITECTURES
|
|
for model in comparison.MODELS for mode, stage in comparison.CONFIGURATIONS}
|
|
|
|
|
|
@pytest.mark.parametrize("damage,message", [("missing", "Missing"), ("duplicate", "Duplicate"),
|
|
("arch", "Unexpected"), ("model", "Unexpected"),
|
|
("pipeline", "Unexpected"), ("mode", "Unexpected")])
|
|
def test_comparison_rejects_bad_matrix_keys(plotters, comparison_csv, tmp_path, damage, message):
|
|
comparison, _ = plotters
|
|
fields, rows = comparison_csv
|
|
if damage == "missing":
|
|
rows.pop()
|
|
elif damage == "duplicate":
|
|
rows[-1] = dict(rows[-2]) # Still 40 rows and 20 per architecture.
|
|
else:
|
|
field = "raptor_pipeline" if damage == "pipeline" else damage
|
|
rows[-1][field] = {"arch": "arch-c", "model": "other", "pipeline": "3", "mode": "other"}[damage]
|
|
with pytest.raises(ValueError, match=message):
|
|
comparison.load_results(write_csv(tmp_path / "results.csv", fields, rows))
|
|
|
|
|
|
def test_matrix_validation_survives_optimized_python(comparison_csv, tmp_path):
|
|
fields, rows = comparison_csv
|
|
path = write_csv(tmp_path / "duplicate.csv", fields, [*rows[:-1], rows[-2]])
|
|
program = """import sys
|
|
from pathlib import Path
|
|
sys.path.insert(0, sys.argv[1])
|
|
import generate_plots
|
|
try:
|
|
generate_plots.load_results(Path(sys.argv[2]))
|
|
except ValueError as exc:
|
|
if 'Duplicate configuration' not in str(exc): raise
|
|
else:
|
|
raise RuntimeError('Invalid matrix accepted with Python -O')
|
|
"""
|
|
subprocess.run([sys.executable, "-O", "-c", program, str(TOOLS / "pimcomp/compare"), str(path)], check=True)
|
|
|
|
|
|
@pytest.mark.parametrize("field,value", [("pimcomp_latency_ms", "999"),
|
|
("pimcomp_throughput_samples_s", "999"), ("pimcomp_power_mw", "999"),
|
|
("pimcomp_energy_pj", "999"), ("pimcomp_functional_validation", "FAIL")])
|
|
def test_repeated_pimcomp_batch_results_must_agree(plotters, comparison_csv, tmp_path, field, value):
|
|
comparison, _ = plotters
|
|
fields, rows = comparison_csv
|
|
row = next(row for row in rows if row["arch"] == "arch-a" and row["model"] == "vgg8"
|
|
and row["raptor_pipeline"] == "4")
|
|
row[field] = value
|
|
with pytest.raises(ValueError, match="inconsistent shared Pimcomp batch results"):
|
|
comparison.load_results(write_csv(tmp_path / "results.csv", fields, rows))
|
|
|
|
|
|
def test_legacy_invalid_ablation_pairs_are_unavailable_without_changing_pass(plotters):
|
|
_, ablation = plotters
|
|
with pytest.warns(UserWarning, match="2 invalid reciprocal"):
|
|
rows = ablation.load_results(TOOLS / "ablation/results_ablation.csv")
|
|
invalid = [row for row in rows if row.model == "googlenet" and row.variant == "no-plan"
|
|
and row.mode == "throughput"]
|
|
assert len(invalid) == 2
|
|
assert all(row.validation == "PASS" and math.isnan(row.metrics["latency"])
|
|
and math.isnan(row.metrics["throughput"]) for row in invalid)
|
|
|
|
|
|
def test_ablation_missing_values_and_legitimate_extremes(plotters, tmp_path):
|
|
_, ablation = plotters
|
|
row = {"variant": "no-plan", "arch": "arch-a", "model": "vgg8", "mode": "throughput",
|
|
"raptor_pipeline": "4", "functional_validation": "FAIL", "latency": "-100.00%",
|
|
"throughput": "+100000000.00%", "power": "NA", "energy": "-100.00%"}
|
|
result = ablation.load_results(write_csv(tmp_path / "extreme.csv", list(row), [row]))[0]
|
|
assert result.validation == "FAIL"
|
|
assert result.metrics["latency"] == result.metrics["energy"] == -100
|
|
assert result.metrics["throughput"] == 100000000
|
|
assert math.isnan(result.metrics["power"])
|
|
|
|
|
|
def test_both_plotters_export_png_and_svg(plotters, tmp_path):
|
|
comparison, ablation = plotters
|
|
comparison.configure_style()
|
|
rows = comparison.load_results(TOOLS / "pimcomp/compare/results_comparison.csv")
|
|
selected = [row for row in rows if row.model == "vgg8" and row.mode == "latency"]
|
|
comparison.draw_absolute(selected, "latency", "Latency (ms)", "comparison-smoke", tmp_path)
|
|
row = ablation.Result("baseline", "arch-a", "vgg8", "latency", 1, "PASS",
|
|
{"latency": math.nan, "power": 0, "energy": 0, "throughput": math.nan})
|
|
ablation.draw_metric([row], "latency", "Latency change (%)", tmp_path)
|
|
for name in ("comparison-smoke", "latency_pipeline1_latency"):
|
|
assert (tmp_path / f"png/{name}.png").read_bytes().startswith(b"\x89PNG\r\n\x1a\n")
|
|
svg = (tmp_path / f"svg/{name}.svg").read_text()
|
|
assert "<svg" in svg
|
|
assert "N/A" in (tmp_path / "svg/latency_pipeline1_latency.svg").read_text()
|