even better more structured throughput scheduling
Validate Operations / validate-operations (push) Has been cancelled
Validate Operations / validate-operations (push) Has been cancelled
plot adjustments validation adjustments
This commit is contained in:
@@ -0,0 +1,75 @@
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
|
||||
from raptor_validation.output_comparison import compare_output_arrays
|
||||
|
||||
|
||||
def test_absolute_floor_does_not_accept_lost_small_output():
|
||||
result = compare_output_arrays(np.zeros(2), np.array([1e-4, -2e-4]), atol=1e-3, rtol=1e-4)
|
||||
assert result.pointwise_passed
|
||||
assert result.relative_l2_error == pytest.approx(1)
|
||||
assert not result.passed
|
||||
|
||||
|
||||
@pytest.mark.parametrize("scale", [1e-200, 1e-4, 1, 1e200])
|
||||
def test_small_relative_roundoff_is_independent_of_scale(scale):
|
||||
expected = np.array([1, -2, 3], dtype=np.float64) * scale
|
||||
result = compare_output_arrays(expected * (1 + 1e-6), expected, atol=1e-3, rtol=1e-4)
|
||||
assert result.passed
|
||||
assert result.relative_l2_error == pytest.approx(1e-6)
|
||||
|
||||
|
||||
def test_norm_agreement_cannot_hide_an_incorrect_small_element():
|
||||
result = compare_output_arrays([1e6, .01], [1e6, 0.0], atol=1e-3, rtol=1e-4)
|
||||
assert result.relative_l2_error < 1e-4
|
||||
assert not result.pointwise_passed
|
||||
assert not result.passed
|
||||
|
||||
|
||||
@pytest.mark.parametrize("actual,expected", [([np.inf], [1.0]), ([1.0], [np.inf]),
|
||||
([np.inf], [np.inf]), ([np.nan], [np.nan])])
|
||||
def test_nonfinite_outputs_never_pass(actual, expected):
|
||||
assert not compare_output_arrays(actual, expected, atol=1e-3, rtol=1e-4).passed
|
||||
|
||||
|
||||
@pytest.mark.parametrize("actual,expected", [([1.0], [[1.0]]), ([1.0], [])])
|
||||
def test_broadcasting_cannot_hide_output_shape_errors(actual, expected):
|
||||
with pytest.raises(ValueError, match="shapes differ"):
|
||||
compare_output_arrays(actual, expected, atol=1e-3, rtol=1e-4)
|
||||
|
||||
|
||||
def test_zero_reference_retains_absolute_check():
|
||||
result = compare_output_arrays([1e-6], [0.0], atol=1e-3, rtol=1e-4)
|
||||
assert result.passed and result.relative_l2_error is None
|
||||
assert not compare_output_arrays([.01], [0.0], atol=1e-3, rtol=1e-4).passed
|
||||
|
||||
|
||||
def test_zero_relative_tolerance_requests_absolute_only_check():
|
||||
result = compare_output_arrays([0.0], [1e-4], atol=1e-3, rtol=0)
|
||||
assert result.passed
|
||||
|
||||
|
||||
def test_integer_outputs_require_exact_agreement_without_float_rounding():
|
||||
expected = np.array([2**53], np.int64)
|
||||
result = compare_output_arrays(expected + 1, expected, atol=1e-3, rtol=1e-4)
|
||||
assert not result.passed and result.max_abs_diff == 1
|
||||
|
||||
|
||||
def test_empty_outputs_match():
|
||||
assert compare_output_arrays([], [], atol=1e-3, rtol=1e-4).passed
|
||||
|
||||
|
||||
@pytest.mark.parametrize("atol,rtol", [(-1, 0), (0, -1), (np.inf, 0), (0, np.nan)])
|
||||
def test_invalid_tolerances_are_rejected(atol, rtol):
|
||||
with pytest.raises(ValueError, match="tolerances"):
|
||||
compare_output_arrays([0.0], [0.0], atol=atol, rtol=rtol)
|
||||
|
||||
|
||||
def test_complex_outputs_are_not_silently_cast_to_real():
|
||||
with pytest.raises(ValueError, match="real numeric"):
|
||||
compare_output_arrays([1 + 2j], [1 + 0j], atol=1e-3, rtol=1e-4)
|
||||
@@ -8,6 +8,8 @@ import os
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
import numpy as np
|
||||
import onnx
|
||||
import pytest
|
||||
|
||||
|
||||
@@ -18,6 +20,93 @@ sys.path.insert(0, str(TOOLS / "pimcomp/compare"))
|
||||
import run_ablation as ablation
|
||||
import run_pimcomp_models as suite
|
||||
import compare_raptor_pimcomp_model as comparator
|
||||
from raptor_validation.validate_one import validate_outputs
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", ["latency", "throughput"])
|
||||
def test_comparison_report_energy_is_per_inference(tmp_path, mode):
|
||||
report = tmp_path / "comparison_report.md"
|
||||
validation = comparator.CompareResult(True, {"output": 1e-7}, relative_l2_errors={"output": 1e-6})
|
||||
metrics = {"average_power_mw": 2, "average_energy_pj": 30,
|
||||
"latency_ms": 4, "average_latency_ms": 4,
|
||||
"throughput": 250, "output_count": 6}
|
||||
comparator.write_report(
|
||||
report, model_path=tmp_path / "model.onnx", hardware={}, steps=[], failures=[],
|
||||
raptor_validation=validation, pimcomp_validation=validation,
|
||||
raptor_perf=metrics, pimcomp_perf=metrics, raptor_instr={}, pimcomp_instr={},
|
||||
raptor_pass_timings={}, pimsim_mode=mode, pimcomp_pipeline="batch",
|
||||
pimcomp_replication="weight")
|
||||
text = report.read_text()
|
||||
assert "Average power (mW) | Energy per inference (pJ/inference)" in text
|
||||
assert "Avg latency" not in text
|
||||
assert "Avg energy" not in text
|
||||
assert metrics["average_energy_pj"] == 30
|
||||
assert "Raptor relative L2 | Pimcomp relative L2" in text
|
||||
assert "1.000000e-06 | 1.000000e-06" in text
|
||||
|
||||
|
||||
@pytest.mark.parametrize("zero_output", [False, True])
|
||||
def test_comparison_frontends_reject_missing_small_outputs(tmp_path, zero_output):
|
||||
expected = np.array([[1e-4, -2e-4]], dtype=np.float32)
|
||||
actual = np.zeros_like(expected) if zero_output else expected * np.float32(1 + 1e-6)
|
||||
np.savetxt(tmp_path / "output0_output.csv", expected.reshape(-1), delimiter=",")
|
||||
actual.tofile(tmp_path / "output.bin")
|
||||
outputs = [(0, "output", 1, [1, 2])]
|
||||
result = comparator.compare_simulator_outputs(
|
||||
tmp_path / "output.bin", outputs, tmp_path, threshold=1e-3, rtol=1e-4)
|
||||
assert result.passed == (not zero_output)
|
||||
assert result.relative_l2_errors["output"] == pytest.approx(1 if zero_output else 1e-6, rel=.1)
|
||||
assert validate_outputs([actual], tmp_path, outputs, 1e-3, 1e-4, False) == (not zero_output)
|
||||
|
||||
|
||||
def test_comparison_preserves_double_reference_precision(tmp_path):
|
||||
expected = np.array([1 + 2**-40], dtype=np.float64)
|
||||
np.savetxt(tmp_path / "output0_output.csv", expected, delimiter=",")
|
||||
expected.tofile(tmp_path / "output.bin")
|
||||
outputs = [(0, "output", 11, [1])]
|
||||
assert comparator.compare_simulator_outputs(
|
||||
tmp_path / "output.bin", outputs, tmp_path, threshold=0, rtol=0).passed
|
||||
assert validate_outputs([expected], tmp_path, outputs, 0, 0, False)
|
||||
|
||||
|
||||
def test_relative_agreement_only_reports_possible_numerical_sensitivity(tmp_path):
|
||||
expected = np.array([1e6, 0], dtype=np.float32)
|
||||
actual = np.array([1e6, .01], dtype=np.float32)
|
||||
np.savetxt(tmp_path / "output0_output.csv", expected, delimiter=",")
|
||||
actual.tofile(tmp_path / "output.bin")
|
||||
result = comparator.compare_simulator_outputs(
|
||||
tmp_path / "output.bin", [(0, "output", 1, [2])], tmp_path, threshold=1e-3, rtol=1e-4)
|
||||
assert not result.passed
|
||||
assert result.relative_l2_errors["output"] < 1e-4
|
||||
assert "possible numerical sensitivity, not proof of correctness" in result.error
|
||||
|
||||
|
||||
def test_comparison_rejects_missing_output_arrays(tmp_path, monkeypatch):
|
||||
outputs = [(0, "output", 1, [1])]
|
||||
monkeypatch.setattr(comparator, "parse_pim_simulator_outputs", lambda *args: [])
|
||||
with pytest.raises(ValueError, match="output count"):
|
||||
comparator.compare_simulator_outputs(tmp_path / "missing.bin", outputs, tmp_path, threshold=1e-3, rtol=1e-4)
|
||||
with pytest.raises(ValueError, match="output count"):
|
||||
validate_outputs([], tmp_path, outputs, 1e-3, 1e-4, False)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("raptor_pass,pimcomp_pass,winner", [
|
||||
(True, True, "pimcomp"), (True, False, "NA"),
|
||||
(False, True, "NA"), (False, False, "NA"),
|
||||
])
|
||||
def test_timing_winner_requires_both_functional_passes(tmp_path, raptor_pass, pimcomp_pass, winner):
|
||||
report = suite.result_dir(tmp_path, "vgg8", "arch-a", "latency", 1) / "pimcomp/comparison_report.json"
|
||||
report.parent.mkdir(parents=True)
|
||||
data = {"validation_policy": suite.VALIDATION_POLICY}
|
||||
for side, passed, latency in [("raptor", raptor_pass, 10), ("pimcomp", pimcomp_pass, 1)]:
|
||||
data[side + "_validation"] = {"status": "done", "passed": passed}
|
||||
data[side + "_performance"] = {"latency_ms": latency, "output_count": 1}
|
||||
report.write_text(json.dumps(data))
|
||||
output = suite.write_results_csv(tmp_path, "arch-a", ["vgg8"], (("latency", 1, "element"),))
|
||||
with output.open() as stream:
|
||||
row = next(csv.DictReader(stream))
|
||||
assert row["better_compiler"] == winner
|
||||
assert row["speedup"] == "10.00" # Diagnostic timing remains available.
|
||||
|
||||
|
||||
def test_ablation_results_stay_in_independent_output_roots(tmp_path):
|
||||
@@ -30,7 +119,7 @@ def test_ablation_results_stay_in_independent_output_roots(tmp_path):
|
||||
path.parent.mkdir(parents=True)
|
||||
row = {"arch": "arch-a", "model": "vgg8", "mode": "latency",
|
||||
"raptor_pipeline": "1", "raptor_functional_validation": "PASS"}
|
||||
row.update({field: 100 if variant.name == "baseline" else value
|
||||
row.update({field: value if variant.name == "baseline" else 100
|
||||
for field, _ in ablation.PERCENTAGE_FIELDS})
|
||||
with path.open("w", newline="") as stream:
|
||||
writer = csv.DictWriter(stream, fieldnames=list(row))
|
||||
@@ -40,11 +129,124 @@ def test_ablation_results_stay_in_independent_output_roots(tmp_path):
|
||||
assert failures == []
|
||||
assert output == root / "results_ablation.csv"
|
||||
outputs.append(output)
|
||||
for output, change in zip(outputs, ("-20.00%", "-40.00%")):
|
||||
for output, change in zip(outputs, ("+20.00%", "+40.00%")):
|
||||
with output.open(newline="") as stream:
|
||||
rows = list(csv.DictReader(stream))
|
||||
assert rows[0]["latency"] == "+0.00%"
|
||||
assert rows[1]["latency"] == change
|
||||
assert len(rows) == 1
|
||||
assert rows[0]["comparison"] == "plan"
|
||||
assert rows[0]["baseline_variant"] == "no-plan"
|
||||
assert rows[0]["latency_gain_percent"] == change
|
||||
|
||||
|
||||
@pytest.mark.parametrize("options,expected", [
|
||||
([], {("baseline", "latency", "1"), ("no-plan", "latency", "1"),
|
||||
("baseline", "throughput", "4"), ("no-plan", "throughput", "4"),
|
||||
("no-sync", "throughput", "4")}),
|
||||
(["--mode", "latency"], {("baseline", "latency", "1"), ("no-plan", "latency", "1")}),
|
||||
(["--pipeline=8"], {(variant, "throughput", "8") for variant in ("baseline", "no-plan", "no-sync")}),
|
||||
(["--mode", "latency", "throughput", "--pipeline", "2"],
|
||||
{(variant, "throughput", "2") for variant in ("baseline", "no-plan", "no-sync")}),
|
||||
])
|
||||
def test_ablation_runs_only_required_variants(options, expected, capsys):
|
||||
import shlex
|
||||
|
||||
assert ablation.main(["--dry-run", *options]) == 0
|
||||
commands = [shlex.split(line) for line in capsys.readouterr().out.splitlines()]
|
||||
actual = set()
|
||||
for command in commands:
|
||||
variant = command[command.index("--ablation-variant") + 1]
|
||||
actual.add((variant, *ablation.case_key(command)))
|
||||
assert "--raptor-only" in command
|
||||
assert ("--raptor-extra-arg=--pim-disable-synchronization" in command) == (variant == "no-sync")
|
||||
assert ("--raptor-extra-arg=--pim-disable-spatial-planning" in command) == (variant == "no-plan")
|
||||
assert actual == expected
|
||||
assert len(commands) == len(expected)
|
||||
|
||||
|
||||
def test_ablation_rejects_combined_disabled_variant():
|
||||
with pytest.raises(SystemExit):
|
||||
ablation.parse_args(["--variant", "no-sync-no-plan"])
|
||||
|
||||
|
||||
def test_ablation_reports_gains_against_each_disabled_feature(tmp_path):
|
||||
variants = ablation.selected_variants(None)
|
||||
for variant in variants:
|
||||
path = ablation.variant_output_dir(tmp_path, variant) / "results_comparison.csv"
|
||||
path.parent.mkdir(parents=True)
|
||||
rows = []
|
||||
for mode, pipeline in (("latency", "1"), ("throughput", "4")):
|
||||
if variant.name == "no-sync" and mode == "latency":
|
||||
continue
|
||||
rows.append({"arch": "arch-b", "model": "vgg8", "mode": mode,
|
||||
"raptor_pipeline": pipeline,
|
||||
"raptor_functional_validation": "FAIL" if variant.name == "no-sync" else "PASS",
|
||||
"raptor_latency_ms": {"baseline": "8", "no-plan": "10", "no-sync": "6.4"}[variant.name],
|
||||
"raptor_throughput_samples_s": {"baseline": "125", "no-plan": "100", "no-sync": "156.25"}[variant.name],
|
||||
"raptor_power_mw": "100", "raptor_energy_pj": "100"})
|
||||
with path.open("w", newline="") as stream:
|
||||
writer = csv.DictWriter(stream, fieldnames=list(rows[0]))
|
||||
writer.writeheader()
|
||||
writer.writerows(rows)
|
||||
output, failures = ablation.aggregate_results(tmp_path, variants)
|
||||
assert not failures
|
||||
with output.open(newline="") as stream:
|
||||
rows = list(csv.DictReader(stream))
|
||||
assert len(rows) == 3
|
||||
latency, plan, sync = rows
|
||||
assert (latency["comparison"], latency["latency_gain_percent"]) == ("plan", "+20.00%")
|
||||
assert (plan["comparison"], plan["throughput_gain_percent"]) == ("plan", "+25.00%")
|
||||
assert (sync["comparison"], sync["throughput_gain_percent"]) == ("sync", "-20.00%")
|
||||
assert plan["baseline_variant"] == "no-plan"
|
||||
assert sync["baseline_variant"] == "no-sync"
|
||||
assert sync["baseline_functional_validation"] == "FAIL"
|
||||
assert sync["functional_validation"] == "PASS"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("variant", [None, "baseline", "no-plan", "no-sync"])
|
||||
def test_unconverged_fast_throughput_is_available_in_comparison_and_ablation(tmp_path, variant):
|
||||
raw = """Fast throughput convergence failed before sim_time; using legacy full-duration throughput result
|
||||
Chip Simulation Result:
|
||||
- output count: 22 samples
|
||||
- throughput: 22 samples/s
|
||||
- average latency: 45.45454545 ms
|
||||
- average power: 300 mW
|
||||
- average energy: 10000000000 pJ/it
|
||||
"""
|
||||
metrics = comparator.parse_pimsim_nn_metrics(raw)
|
||||
assert metrics["fast_throughput_converged"] is False
|
||||
assert metrics["throughput_measurement"] == "full_duration_fallback"
|
||||
assert "unconverged" in comparator.perf_status(metrics)
|
||||
report = suite.result_dir(tmp_path, "vgg8", "arch-b", "throughput", 4, variant) / "pimcomp/comparison_report.json"
|
||||
report.parent.mkdir(parents=True)
|
||||
data = {"validation_policy": suite.VALIDATION_POLICY}
|
||||
for side in ("raptor", "pimcomp"):
|
||||
data[side + "_validation"] = {"status": "done", "passed": True}
|
||||
# Older reports can contain only the raw simulator output.
|
||||
data[side + "_performance"] = {"raw_output": raw, "fast_throughput_converged": False}
|
||||
report.write_text(json.dumps(data))
|
||||
output = suite.write_results_csv(tmp_path, "arch-b", ["vgg8"],
|
||||
(("throughput", 4, "batch"),), ablation_variant=variant)
|
||||
with output.open(newline="") as stream:
|
||||
row = next(csv.DictReader(stream))
|
||||
for side in ("raptor", "pimcomp"):
|
||||
assert float(row[side + "_throughput_samples_s"]) == 22
|
||||
assert float(row[side + "_latency_ms"]) == pytest.approx(1000 / 22)
|
||||
if variant is not None:
|
||||
enabled = ablation.variant_output_dir(tmp_path, ablation.VARIANT_BY_NAME["baseline"]) / "results_comparison.csv"
|
||||
enabled.parent.mkdir(parents=True, exist_ok=True)
|
||||
enabled.write_text(output.read_text())
|
||||
aggregate, failures = ablation.aggregate_results(tmp_path, ablation.selected_variants([variant]))
|
||||
assert not failures
|
||||
with aggregate.open(newline="") as stream:
|
||||
for gain in csv.DictReader(stream):
|
||||
assert gain["throughput_gain_percent"] == "+0.00%"
|
||||
|
||||
|
||||
def test_fast_throughput_convergence_metadata():
|
||||
metrics = comparator.parse_pimsim_nn_metrics("Fast throughput converged; stopping early\nthroughput: 100 samples/s")
|
||||
assert metrics["fast_throughput_converged"] is True
|
||||
assert metrics["throughput_measurement"] == "converged_windows"
|
||||
assert metrics["throughput"] == 100
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@@ -56,7 +258,9 @@ def recorded_failure(tmp_path):
|
||||
pimcomp_config=config, pimsim_mode="throughput", pimsim_time_ms=1000,
|
||||
pimcomp_pipeline="batch", pimcomp_replication="GA", pimcomp_model_name=None,
|
||||
seed=0, batch_size=64, threshold=1e-3, rtol=1e-4, no_fast=False,
|
||||
skip_pimsim_nn=False,
|
||||
skip_pimsim_nn=False, pimcomp_dir=tmp_path / "pimcomp",
|
||||
pimsim_nn_build_dir=tmp_path / "oracle", pim_simulator_dir=tmp_path / "functional",
|
||||
raptor_path=tmp_path / "raptor", raptor_extra_arg=["--pipeline=2"],
|
||||
)
|
||||
hardware = {"core_count": 168, "crossbar_size": 128}
|
||||
context = comparator.pimcomp_reuse_context(args, model, functional, hardware)
|
||||
@@ -66,7 +270,7 @@ def recorded_failure(tmp_path):
|
||||
"error": "Unsupported model"},
|
||||
"pimcomp_performance": {"skipped": True, "reason": "Compilation failed"},
|
||||
"pimcomp_instruction_summary": comparator.empty_instruction_summary("No artifact"),
|
||||
"paths": {}, "steps": [],
|
||||
"paths": {}, "steps": [], "artifact_signatures": {"pimcomp": {}},
|
||||
"failures": [{"stage": "Compile Pimcomp frontend", "error": "Unsupported model"}],
|
||||
}
|
||||
path = tmp_path / "comparison_report.json"
|
||||
@@ -84,6 +288,117 @@ def test_failed_pimcomp_report_needs_no_simulation_artifacts(recorded_failure):
|
||||
assert failures == report["failures"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tool", ["compiler", "oracle", "exporter"])
|
||||
def test_reuse_checks_tool_content(recorded_failure, tool):
|
||||
path, report, args, model, functional, hardware = recorded_failure
|
||||
original = report["pimcomp_reuse_context"]["files"][tool]["path"]
|
||||
tool_path = Path(original)
|
||||
tool_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
tool_path.write_bytes(b"changed executable or exporter")
|
||||
expected = comparator.pimcomp_reuse_context(args, model, functional, hardware)
|
||||
with pytest.raises(ValueError, match="Cannot reuse Pimcomp report"):
|
||||
comparator.load_reused_pimcomp_report(path, expected)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mutation", ["instructions", "weights", "missing"])
|
||||
def test_reuse_checks_artifact_content(recorded_failure, mutation):
|
||||
path, report, *_ = recorded_failure
|
||||
artifact = path.parent / "artifact"
|
||||
artifact.mkdir()
|
||||
(artifact / "core_0.json").write_text('[{"op":"nop"}]')
|
||||
(artifact / "weights.bin").write_bytes(b"weights")
|
||||
report["paths"]["pimcomp_exported_pim"] = str(artifact)
|
||||
report["artifact_signatures"]["pimcomp"] = comparator.artifact_signatures(report, "pimcomp")
|
||||
path.write_text(json.dumps(report))
|
||||
assert comparator.load_reused_pimcomp_report(path, report["pimcomp_reuse_context"])
|
||||
if mutation == "missing":
|
||||
(artifact / "core_0.json").unlink()
|
||||
elif mutation == "weights":
|
||||
(artifact / "weights.bin").write_bytes(b"wrong weights")
|
||||
else:
|
||||
(artifact / "core_0.json").write_text('[{"op":"wait"}]')
|
||||
with pytest.raises(ValueError, match="artifacts differ"):
|
||||
comparator.load_reused_pimcomp_report(path, report["pimcomp_reuse_context"])
|
||||
|
||||
|
||||
def test_raptor_context_covers_pipeline_and_ignores_pimcomp_pipeline(recorded_failure):
|
||||
_, _, args, model, functional, hardware = recorded_failure
|
||||
expected = comparator.experiment_context(args, model, functional, hardware, "raptor")
|
||||
args.pimcomp_pipeline = "element"
|
||||
assert expected == comparator.experiment_context(args, model, functional, hardware, "raptor")
|
||||
args.raptor_extra_arg = ["--pipeline=8"]
|
||||
assert expected != comparator.experiment_context(args, model, functional, hardware, "raptor")
|
||||
|
||||
|
||||
def test_reuse_rejects_previous_validation_policy(recorded_failure):
|
||||
path, report, *_ = recorded_failure
|
||||
context = copy.deepcopy(report["pimcomp_reuse_context"])
|
||||
del report["pimcomp_reuse_context"]["validation_policy"]
|
||||
path.write_text(json.dumps(report))
|
||||
with pytest.raises(ValueError, match="Cannot reuse Pimcomp report"):
|
||||
comparator.load_reused_pimcomp_report(path, context)
|
||||
|
||||
|
||||
def test_raptor_reuse_rejects_previous_validation_policy(recorded_failure, monkeypatch):
|
||||
path, _, _, model, _, _ = recorded_failure
|
||||
path.write_text('{"validation_policy": "legacy"}')
|
||||
monkeypatch.setattr(comparator, "load_effective_hardware", lambda args: {})
|
||||
monkeypatch.setattr(comparator, "load_model_inputs", lambda *args: ([], [], []))
|
||||
monkeypatch.setattr(sys, "argv", [
|
||||
"compare_raptor_pimcomp_model.py", "--model", str(model),
|
||||
"--out-dir", str(path.parent / "out"), "--pimcomp-config", str(path.parent / "config.json"),
|
||||
"--reuse-raptor-report", str(path),
|
||||
])
|
||||
with pytest.raises(ValueError, match="validation policy differs"):
|
||||
comparator.main()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("tolerances", [None, {"atol": 0.01, "rtol": 0.0001},
|
||||
{"atol": 0.001, "rtol": 0.01}])
|
||||
def test_raptor_reuse_checks_actual_tolerances(recorded_failure, monkeypatch, tolerances):
|
||||
path, _, _, model, _, _ = recorded_failure
|
||||
path.write_text(json.dumps({"validation_policy": comparator.VALIDATION_POLICY,
|
||||
"validation_tolerances": tolerances}))
|
||||
monkeypatch.setattr(comparator, "load_effective_hardware", lambda args: {})
|
||||
monkeypatch.setattr(comparator, "load_model_inputs", lambda *args: ([], [], []))
|
||||
monkeypatch.setattr(sys, "argv", [
|
||||
"compare_raptor_pimcomp_model.py", "--model", str(model),
|
||||
"--out-dir", str(path.parent / "out"), "--pimcomp-config", str(path.parent / "config.json"),
|
||||
"--reuse-raptor-report", str(path),
|
||||
])
|
||||
with pytest.raises(ValueError, match="validation tolerances differ"):
|
||||
comparator.main()
|
||||
|
||||
|
||||
def test_failed_raptor_compilation_is_not_reported_as_skipped(tmp_path, monkeypatch):
|
||||
model = tmp_path / "model.onnx"
|
||||
model.write_bytes(b"test model")
|
||||
runner = tmp_path / "runner"
|
||||
runner.touch()
|
||||
monkeypatch.setattr(comparator, "load_effective_hardware", lambda args: {"core_count": 1})
|
||||
monkeypatch.setattr(comparator, "load_model_inputs", lambda *args:
|
||||
([(0, "input", 1, [1])], [(0, "output", 1, [1])], [np.zeros(1, np.float32)]))
|
||||
monkeypatch.setattr(comparator, "ensure_reference_runner", lambda *args: runner)
|
||||
monkeypatch.setattr(comparator, "generate_reference_outputs", lambda *args, **kwargs: tmp_path)
|
||||
|
||||
def fail_compile(*args):
|
||||
raise RuntimeError("unsupported test operation")
|
||||
|
||||
monkeypatch.setattr(comparator, "compile_raptor_target", fail_compile)
|
||||
out = tmp_path / "out"
|
||||
monkeypatch.setattr(sys, "argv", [
|
||||
"compare_raptor_pimcomp_model.py", "--model", str(model), "--out-dir", str(out),
|
||||
"--raptor-only", "--skip-pimsim-nn", "--fail-on-error",
|
||||
])
|
||||
with pytest.raises(SystemExit) as exit_code:
|
||||
comparator.main()
|
||||
assert exit_code.value.code == 1
|
||||
report = json.loads((out / "pimcomp/comparison_report.json").read_text())
|
||||
assert report["raptor_validation"]["status"] == "failed"
|
||||
assert not report["raptor_validation"]["passed"]
|
||||
assert "unsupported test operation" in report["failures"][0]["error"]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("setting,value", [
|
||||
("pimsim_mode", "latency"), ("pimcomp_pipeline", "element"),
|
||||
("pimsim_time_ms", 2000), ("pimcomp_replication", "balance"),
|
||||
@@ -177,6 +492,65 @@ def test_missing_pimcomp_report_is_not_reused(tmp_path):
|
||||
assert not suite.pimcomp_report_ready(tmp_path / "output", None)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model", ["yolo11n", "googlenet"])
|
||||
def test_comparison_uses_same_model_for_reference_and_execution(tmp_path, monkeypatch, model):
|
||||
commands = []
|
||||
monkeypatch.setattr(suite, "run", lambda command, **kwargs: commands.append(command) or 0)
|
||||
monkeypatch.setattr(sys, "argv", [
|
||||
"run_pimcomp_models.py", "--out-dir", str(tmp_path), "--models", model,
|
||||
"--archs", "arch-b", "--mode", "throughput", "--pipeline", "8",
|
||||
"--raptor-only", "--dry-run",
|
||||
])
|
||||
assert suite.main() == 0
|
||||
reference, comparison = commands
|
||||
expected = str(suite.MODELS[model])
|
||||
assert reference[reference.index("--model") + 1] == expected
|
||||
assert comparison[comparison.index("--model") + 1] == expected
|
||||
assert comparison[comparison.index("--functional-model") + 1] == expected
|
||||
assert Path(comparison[comparison.index("--common-dir") + 1]).name == "common-full-model"
|
||||
|
||||
|
||||
def test_yolo_uses_full_model_without_moving_comparison_artifacts():
|
||||
assert suite.MODELS["yolo11n"] == suite.REPO / "validation/networks/yolo11n/depth_51/yolo11n_depth_51.onnx"
|
||||
assert suite.model_dir(None, "yolo11n") == suite.SUITE / "yolo11n"
|
||||
assert suite.common_dir(None, "yolo11n") == suite.SUITE / "yolo11n/artifacts/common-full-model"
|
||||
|
||||
|
||||
def test_googlenet_removes_only_lrn():
|
||||
original = onnx.load(suite.SUITE / "googlenet/googlenet-12.onnx")
|
||||
actual = onnx.load(suite.MODELS["googlenet"])
|
||||
removed = {node.output[0]: node.input[0] for node in original.graph.node if node.op_type == "LRN"}
|
||||
assert len(removed) == 2
|
||||
retained = []
|
||||
for node in original.graph.node:
|
||||
if node.op_type == "LRN":
|
||||
continue
|
||||
for i, value in enumerate(node.input):
|
||||
while value in removed:
|
||||
value = removed[value]
|
||||
node.input[i] = value
|
||||
retained.append(copy.deepcopy(node))
|
||||
del original.graph.node[:]
|
||||
original.graph.node.extend(retained)
|
||||
assert actual.SerializeToString() == original.SerializeToString()
|
||||
assert sum(node.op_type == "Softmax" for node in actual.graph.node) == 1
|
||||
assert [output.name for output in actual.graph.output] == ["prob_1"]
|
||||
onnx.checker.check_model(actual)
|
||||
assert suite.common_dir(None, "googlenet").name == "common-full-model"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("model,counts", [
|
||||
("yolo11n", {"Softmax": 2, "Split": 11, "Resize": 2}),
|
||||
("resnet18", {"BatchNormalization": 20, "Conv": 20}),
|
||||
("resnet34", {"BatchNormalization": 36, "Conv": 36}),
|
||||
("vgg8", {"Conv": 6, "Gemm": 2, "Relu": 7, "MaxPool": 3}),
|
||||
])
|
||||
def test_comparison_models_retain_required_operations(model, counts):
|
||||
graph = onnx.load(suite.MODELS[model]).graph
|
||||
for operation, expected in counts.items():
|
||||
assert sum(node.op_type == operation for node in graph.node) == expected
|
||||
|
||||
|
||||
def test_missing_pimcomp_report_is_rejected_at_load(recorded_failure):
|
||||
path, report, *_ = recorded_failure
|
||||
with pytest.raises(ValueError, match="Cannot reuse Pimcomp report"):
|
||||
@@ -341,7 +715,8 @@ def test_zero_outputs_preserve_only_measured_average_power():
|
||||
def test_functional_pass_survives_unavailable_performance(tmp_path):
|
||||
report = suite.result_dir(tmp_path, "vgg8", "arch-a", "throughput", 4) / "pimcomp/comparison_report.json"
|
||||
report.parent.mkdir(parents=True)
|
||||
report.write_text(json.dumps({"raptor_validation": {"status": "done", "passed": True},
|
||||
report.write_text(json.dumps({"validation_policy": suite.VALIDATION_POLICY,
|
||||
"raptor_validation": {"status": "done", "passed": True},
|
||||
"raptor_performance": {"error": "timeout", "throughput": 0}}))
|
||||
output = suite.write_results_csv(tmp_path, "arch-a", ["vgg8"], (("throughput", 4, "batch"),))
|
||||
with output.open(newline="") as stream:
|
||||
@@ -350,6 +725,15 @@ def test_functional_pass_survives_unavailable_performance(tmp_path):
|
||||
assert all(row[field] == "NA" for field, _ in ablation.PERCENTAGE_FIELDS)
|
||||
|
||||
|
||||
def test_aggregate_rejects_previous_validation_policy(tmp_path):
|
||||
report = suite.result_dir(tmp_path, "vgg8", "arch-a", "throughput", 4) / "pimcomp/comparison_report.json"
|
||||
report.parent.mkdir(parents=True)
|
||||
report.write_text(json.dumps({"raptor_validation": {"status": "done", "passed": True}}))
|
||||
with pytest.raises(ValueError, match="different validation policy"):
|
||||
suite.write_results_csv(tmp_path, "arch-a", ["vgg8"], (("throughput", 4, "batch"),))
|
||||
assert not (tmp_path / "results_comparison.csv").exists()
|
||||
|
||||
|
||||
def test_ablation_aggregate_keeps_pass_with_zero_performance(tmp_path):
|
||||
variants = ablation.selected_variants(["no-plan"])
|
||||
for variant in variants:
|
||||
@@ -368,5 +752,7 @@ def test_ablation_aggregate_keeps_pass_with_zero_performance(tmp_path):
|
||||
assert not failures and output == destination
|
||||
with output.open(newline="") as stream:
|
||||
rows = list(csv.DictReader(stream))
|
||||
assert rows[1]["functional_validation"] == "PASS"
|
||||
assert all(rows[1][metric] == "NA" for _, metric in ablation.PERCENTAGE_FIELDS)
|
||||
assert len(rows) == 1
|
||||
assert rows[0]["functional_validation"] == "PASS"
|
||||
assert rows[0]["baseline_functional_validation"] == "PASS"
|
||||
assert all(rows[0][f"{metric}_gain_percent"] == "NA" for _, metric in ablation.PERCENTAGE_FIELDS)
|
||||
|
||||
@@ -53,13 +53,167 @@ def test_comparison_plot_cli_input_paths(comparison, tmp_path, monkeypatch, cust
|
||||
raise InputCaptured
|
||||
|
||||
monkeypatch.setattr(comparison, "load_results", capture)
|
||||
monkeypatch.setattr(comparison, "configure_style", lambda: None)
|
||||
monkeypatch.setattr(comparison, "configure_style", lambda **kwargs: None)
|
||||
monkeypatch.setattr(sys, "argv", ["generate_plots.py", *(["--input", str(expected)] if custom else [])])
|
||||
with pytest.raises(InputCaptured):
|
||||
comparison.main()
|
||||
assert seen == [expected]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("custom", [False, True])
|
||||
def test_comparison_plot_cli_output_paths(comparison, tmp_path, monkeypatch, custom):
|
||||
expected = tmp_path / "custom" if custom else TOOLS / "pimcomp/compare/plots/default"
|
||||
seen = []
|
||||
|
||||
class OutputCaptured(Exception):
|
||||
pass
|
||||
|
||||
def capture(rows, output):
|
||||
seen.append(output)
|
||||
raise OutputCaptured
|
||||
|
||||
monkeypatch.setattr(comparison, "draw_validation", capture)
|
||||
monkeypatch.setattr(sys, "argv", ["generate_plots.py", *(["--output-dir", str(expected)] if custom else [])])
|
||||
with pytest.raises(OutputCaptured):
|
||||
comparison.main()
|
||||
assert seen == [expected]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("flags,colors,anonymize", [([], True, False),
|
||||
(["--colors", "--no-anonymize"], True, False), (["--no-colors"], False, False),
|
||||
(["--anonymize"], True, True), (["--no-colors", "--anonymize"], False, True),
|
||||
(["--no-colors", "--colors", "--anonymize", "--no-anonymize"], True, False)])
|
||||
def test_plot_display_flags(comparison, monkeypatch, flags, colors, anonymize):
|
||||
captured = []
|
||||
|
||||
class StyleCaptured(Exception):
|
||||
pass
|
||||
|
||||
def capture(**options):
|
||||
captured.append(options)
|
||||
raise StyleCaptured
|
||||
|
||||
monkeypatch.setattr(comparison, "configure_style", capture)
|
||||
monkeypatch.setattr(sys, "argv", ["generate_plots.py", *flags])
|
||||
with pytest.raises(StyleCaptured):
|
||||
comparison.main()
|
||||
assert captured == [{"colors": colors, "anonymize": anonymize}]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("flags,borders", [([], True), (["--marker-borders"], True),
|
||||
(["--no-marker-borders"], False), (["--no-marker-borders", "--marker-borders"], True)])
|
||||
def test_marker_border_cli_flags(comparison, monkeypatch, flags, borders):
|
||||
captured = []
|
||||
|
||||
def capture(*args, marker_borders):
|
||||
captured.append(marker_borders)
|
||||
|
||||
for name in ("draw_validation", "draw_metric", "draw_absolute"):
|
||||
monkeypatch.setattr(comparison, name, lambda *args, **kwargs: None)
|
||||
for name in ("draw_compact_summary", "draw_pipeline_effect"):
|
||||
monkeypatch.setattr(comparison, name, capture)
|
||||
monkeypatch.setattr(sys, "argv", ["generate_plots.py", *flags])
|
||||
comparison.main()
|
||||
assert captured == [borders, borders]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("borders", [True, False])
|
||||
def test_marker_borders_reach_data_legends_and_svg(comparison, tmp_path, monkeypatch, borders):
|
||||
from xml.etree import ElementTree as ET
|
||||
from matplotlib.lines import Line2D
|
||||
from matplotlib.colors import to_rgba
|
||||
|
||||
comparison.configure_style()
|
||||
rows = comparison.load_results(TOOLS / "pimcomp/compare/results_comparison.csv")
|
||||
save = comparison.save_figure
|
||||
|
||||
def capture(fig, name, output):
|
||||
try:
|
||||
size = 6.5 if name == "performance_energy_compact" else 7
|
||||
for ax in fig.axes:
|
||||
points = [line for line in ax.lines if line.get_marker() in ("o", "s", "^", "D")]
|
||||
assert points
|
||||
for point in points:
|
||||
assert point.get_markersize() == size
|
||||
assert (point.get_markeredgewidth() > 0) == borders
|
||||
assert to_rgba(point.get_markerfacecolor())[3] == 1
|
||||
if not borders:
|
||||
assert to_rgba(point.get_markerfacecolor()) == to_rgba(point.get_markeredgecolor())
|
||||
handles = [handle for handle in fig.legends[0].legend_handles if isinstance(handle, Line2D)]
|
||||
assert handles
|
||||
for handle in handles:
|
||||
assert handle.get_markersize() == size
|
||||
assert (handle.get_markeredgewidth() > 0) == borders
|
||||
if not borders:
|
||||
assert to_rgba(handle.get_markerfacecolor()) not in ((1, 1, 1, 1), (0, 0, 0, 0))
|
||||
assert to_rgba(handle.get_markerfacecolor()) == to_rgba(handle.get_markeredgecolor())
|
||||
save(fig, name, output)
|
||||
finally:
|
||||
comparison.plt.close(fig)
|
||||
root = ET.parse(output / f"svg/{name}.svg")
|
||||
paths = [node for node in root.findall('.//{http://www.w3.org/2000/svg}defs/{http://www.w3.org/2000/svg}path')
|
||||
if node.get("id", "").startswith("m")]
|
||||
assert paths
|
||||
assert all(("stroke:" in node.get("style", "")) == borders for node in paths)
|
||||
|
||||
monkeypatch.setattr(comparison, "save_figure", capture)
|
||||
comparison.draw_compact_summary(rows, tmp_path, marker_borders=borders)
|
||||
comparison.draw_pipeline_effect(rows, tmp_path, marker_borders=borders)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("colors", [True, False])
|
||||
@pytest.mark.parametrize("anonymize", [True, False])
|
||||
def test_plot_display_variant_exports(comparison, tmp_path, colors, anonymize):
|
||||
import re
|
||||
from PIL import Image, ImageChops
|
||||
|
||||
try:
|
||||
comparison.configure_style(colors=colors, anonymize=anonymize)
|
||||
rows = comparison.load_results(TOOLS / "pimcomp/compare/results_comparison.csv")
|
||||
comparison.draw_compact_summary(rows, tmp_path)
|
||||
selected = comparison.select_compact_speedups([row for row in rows if row.model != "yolo11n"])
|
||||
comparison.draw_metric(selected, "speedup", "Speedup (x)", "speedup_compact_comparison",
|
||||
tmp_path, compact=True)
|
||||
for name in ("performance_energy_compact", "speedup_compact_comparison"):
|
||||
svg = (tmp_path / f"svg/{name}.svg").read_text()
|
||||
if name == "performance_energy_compact":
|
||||
assert ("Anonimized better" in svg) == anonymize
|
||||
assert ("Raptor better" in svg) == (not anonymize)
|
||||
svg_colors = re.findall(r"(?:fill|stroke):\s*#([0-9a-fA-F]{6})", svg)
|
||||
assert svg_colors
|
||||
assert any(color[:2] != color[2:4] or color[2:4] != color[4:] for color in svg_colors) == colors
|
||||
with Image.open(tmp_path / f"png/{name}.png") as picture:
|
||||
red, green, blue = picture.convert("RGB").split()
|
||||
chromatic = (ImageChops.difference(red, green).getbbox() is not None or
|
||||
ImageChops.difference(green, blue).getbbox() is not None)
|
||||
assert chromatic == colors
|
||||
assert (tmp_path / f"pdf/{name}.pdf").read_bytes().startswith(b"%PDF-")
|
||||
finally:
|
||||
comparison.configure_style()
|
||||
|
||||
|
||||
def test_anonymization_reaches_all_plot_labels(comparison, monkeypatch):
|
||||
from matplotlib.text import Text
|
||||
|
||||
captured = {}
|
||||
|
||||
def capture(fig, name, output):
|
||||
captured[name] = [text.get_text() for text in fig.findobj(Text)]
|
||||
comparison.plt.close(fig)
|
||||
|
||||
monkeypatch.setattr(comparison, "save_figure", capture)
|
||||
monkeypatch.setattr(sys, "argv", ["generate_plots.py", "--anonymize"])
|
||||
try:
|
||||
comparison.main()
|
||||
assert len(captured) == 30
|
||||
assert all("Raptor" not in label for labels in captured.values() for label in labels)
|
||||
assert "Anonimized pipeline stages" in captured["raptor_pipeline_throughput"]
|
||||
assert "Anonimized better →" in captured["performance_energy_compact"]
|
||||
assert "Anonimized\n2 stages" in captured["validation_status"]
|
||||
finally:
|
||||
comparison.configure_style()
|
||||
|
||||
|
||||
def test_exact_comparison_configuration_matrix(comparison, comparison_csv, tmp_path):
|
||||
fields, rows = comparison_csv
|
||||
results = comparison.load_results(write_csv(tmp_path / "results.csv", fields, rows))
|
||||
@@ -185,7 +339,7 @@ def test_compact_validation_combines_pipeline_statuses(comparison, tmp_path, mon
|
||||
assert [label.get_text() for label in ax.get_yticklabels()] == ([
|
||||
"VGG-8", "ResNet-18", "ResNet-34", "GoogLeNet", "YOLO11n"] if index == 0 else [])
|
||||
assert len(ax.patches) == len(ax.texts) == 4 * len(comparison.MODELS)
|
||||
assert [label.get_text() for label in ax.get_xticklabels()] == ["Latency", "Throughput"]
|
||||
assert [label.get_text() for label in ax.get_xticklabels()] == ["Latency-mode", "Throughput-mode"]
|
||||
assert all(text.get_fontsize() == 12 and text.get_weight() == "normal" for text in ax.texts)
|
||||
assert all(text.get_color() == "#1D2530" for text in ax.texts)
|
||||
ax = fig.axes[0]
|
||||
@@ -211,15 +365,21 @@ def test_compact_speedups_select_best_pipeline(comparison, rates, stage):
|
||||
|
||||
@pytest.mark.parametrize("diverging", [False, True])
|
||||
@pytest.mark.parametrize("metric,ylabel", [("speedup", "Speedup (x)"),
|
||||
("energy", "Energy-per-inference advantage (x)"), ("power", "Average power advantage (x)")])
|
||||
("energy", "Energy reduction (x)"), ("power", "Average power reduction (x)")])
|
||||
def test_compact_metric_bars_and_units(comparison, tmp_path, monkeypatch, diverging, metric, ylabel):
|
||||
from matplotlib.colors import to_rgba
|
||||
|
||||
template = comparison.load_results(TOOLS / "pimcomp/compare/results_comparison.csv")[0]
|
||||
rows = [replace(template, arch=arch, mode=mode, pipeline=stage, latency_speedup=0.5,
|
||||
energy=0.5 if mode == "latency" else 3, power=0.5 if mode == "latency" else 3,
|
||||
raptor_throughput=300, pimcomp_throughput=100, pimcomp_validation="FAIL")
|
||||
for arch in comparison.ARCHITECTURES for mode, stage in (("latency", 1), ("throughput", 4))]
|
||||
rows = [replace(template, model=model, arch=arch, mode=mode, pipeline=stage,
|
||||
latency_speedup=0.5 if arch == "arch-a" else 0.25,
|
||||
energy=(0.5 if arch == "arch-a" else 0.25) if mode == "latency" else
|
||||
(3 if arch == "arch-a" else 4),
|
||||
power=(0.5 if arch == "arch-a" else 0.25) if mode == "latency" else
|
||||
(3 if arch == "arch-a" else 4),
|
||||
raptor_throughput=300 if arch == "arch-a" else 400,
|
||||
pimcomp_throughput=100, pimcomp_validation="FAIL")
|
||||
for arch in reversed(comparison.ARCHITECTURES) for model in ("resnet18", "vgg8")
|
||||
for mode, stage in (("throughput", 4), ("latency", 1))]
|
||||
captured = []
|
||||
monkeypatch.setattr(comparison, "save_figure", lambda fig, name, path: captured.append(fig))
|
||||
comparison.draw_metric(rows, metric, ylabel, "compact", tmp_path,
|
||||
@@ -227,30 +387,84 @@ def test_compact_metric_bars_and_units(comparison, tmp_path, monkeypatch, diverg
|
||||
fig = captured[0]
|
||||
try:
|
||||
assert fig.get_size_inches() == pytest.approx((8.5, 4.4))
|
||||
expected_legend = {
|
||||
"speedup": ["Single-inference latency", "Best-case throughput"],
|
||||
"energy": ["Single-inference energy", "Best-case throughput energy"],
|
||||
"power": ["Single-inference power", "Best-case throughput power"],
|
||||
expected_panels = {
|
||||
"speedup": ["Latency-mode", "Throughput-mode"],
|
||||
"energy": ["Latency-mode energy", "Throughput-mode energy"],
|
||||
"power": ["Latency-mode power", "Throughput-mode power"],
|
||||
}
|
||||
assert [text.get_text() for text in fig.legends[0].get_texts()] == expected_legend[metric]
|
||||
assert [text.get_text() for text in fig.legends[0].get_texts()] == ["Arch-A", "Arch-B"]
|
||||
for index, ax in enumerate(fig.axes):
|
||||
latency, throughput = (container.patches[0] for container in ax.containers)
|
||||
bars = [container.patches[0] for container in ax.containers]
|
||||
assert len(bars) == 4
|
||||
assert ax.get_xlabel() == expected_panels[metric][index]
|
||||
assert [text.get_text() for text in ax.get_xticklabels()] == ["VGG-8", "ResNet-18"]
|
||||
assert list(ax.get_xticks()) == [0.5, 3.5]
|
||||
lower, upper = ax.get_xlim()
|
||||
assert latency.get_x() - lower == pytest.approx(0.4)
|
||||
assert upper - throughput.get_x() - throughput.get_width() == pytest.approx(0.4)
|
||||
assert latency.get_x() + latency.get_width() == throughput.get_x()
|
||||
assert latency.get_facecolor() == to_rgba(comparison.METRIC_STYLES["latency"][0])
|
||||
assert throughput.get_facecolor() == to_rgba(comparison.METRIC_STYLES["throughput"][0])
|
||||
assert latency.get_edgecolor() == to_rgba(comparison.METRIC_STYLES["latency"][1])
|
||||
assert throughput.get_edgecolor() == to_rgba(comparison.METRIC_STYLES["throughput"][1])
|
||||
assert not latency.get_hatch() and throughput.get_hatch() == "//"
|
||||
assert latency.get_height() == (-1 if diverging else 0.5)
|
||||
assert throughput.get_height() == (2 if diverging else 3)
|
||||
assert bars[0].get_x() - lower == pytest.approx(0.4)
|
||||
assert upper - bars[-1].get_x() - bars[-1].get_width() == pytest.approx(0.4)
|
||||
for offset in (0, 2):
|
||||
arch_a, arch_b = bars[offset:offset + 2]
|
||||
assert arch_a.get_x() + arch_a.get_width() == arch_b.get_x()
|
||||
for bar, arch in zip((arch_a, arch_b), comparison.ARCHITECTURES):
|
||||
fill, border, pattern = comparison.ARCH_STYLES[arch]
|
||||
assert bar.get_facecolor() == to_rgba(fill)
|
||||
assert bar.get_edgecolor() == to_rgba(border)
|
||||
assert bar.get_hatch() == pattern
|
||||
assert arch_a.get_height() == ((-1 if diverging else 0.5) if index == 0 else (2 if diverging else 3))
|
||||
assert arch_b.get_height() == ((-3 if diverging else 0.25) if index == 0 else (3 if diverging else 4))
|
||||
assert bars[1].get_x() + bars[1].get_width() < bars[2].get_x()
|
||||
assert ax.get_ylabel() == (ylabel if index == 0 else "")
|
||||
assert bool(ax.get_yticklabels()) == (index == 0)
|
||||
assert all("×" not in label.get_text() for label in ax.get_yticklabels())
|
||||
assert [text.get_text() for text in ax.texts] == [
|
||||
"2.00×" if diverging else "0.50×", "P. INVALID", "3.00×", "P. INVALID"]
|
||||
values = (["2.00×", "4.00×"] if diverging else ["0.50×", "0.25×"]) if index == 0 else ["3.00×", "4.00×"]
|
||||
assert [text.get_text() for text in ax.texts] == [value for _ in range(2) for number in values
|
||||
for value in (number, "P. INVALID")]
|
||||
finally:
|
||||
comparison.plt.close(fig)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("minimum,expected", [(0.4, 0.25), (0.5, 0.5), (0.8, 0.5)])
|
||||
@pytest.mark.parametrize("compact", [False, True])
|
||||
def test_ratio_bar_minimum_keeps_values_and_labels_visible(comparison, tmp_path, monkeypatch,
|
||||
minimum, expected, compact):
|
||||
template = comparison.load_results(TOOLS / "pimcomp/compare/results_comparison.csv")[0]
|
||||
rows = [replace(template, arch=arch, mode=mode, pipeline=stage,
|
||||
power=minimum if arch == "arch-a" else 2 if mode == "latency" else math.nan)
|
||||
for arch in comparison.ARCHITECTURES for mode, stage in (("latency", 1), ("throughput", 4))]
|
||||
captured = []
|
||||
monkeypatch.setattr(comparison, "save_figure", lambda fig, *args: captured.append(fig))
|
||||
comparison.draw_metric(rows, "power", "Average power reduction (x)", "power", tmp_path, compact=compact)
|
||||
fig = captured[0]
|
||||
try:
|
||||
fig.canvas.draw()
|
||||
renderer = fig.canvas.get_renderer()
|
||||
assert fig.axes[0].get_ylim() == fig.axes[1].get_ylim()
|
||||
for index, ax in enumerate(fig.axes):
|
||||
assert ax.get_ylim()[0] == expected
|
||||
if index == 0:
|
||||
assert f"{expected:g}" in [text.get_text() for text in ax.get_yticklabels(which="both")]
|
||||
else:
|
||||
assert not ax.get_yticklabels(which="both")
|
||||
bounds = ax.get_window_extent(renderer)
|
||||
for text in ax.texts:
|
||||
extent = text.get_window_extent(renderer)
|
||||
assert bounds.y0 <= extent.y0 <= extent.y1 <= bounds.y1
|
||||
assert any(text.get_text() == "N/A" for ax in fig.axes for text in ax.texts)
|
||||
finally:
|
||||
comparison.plt.close(fig)
|
||||
|
||||
|
||||
def test_compact_summary_uses_lower_minimum_when_needed(comparison, tmp_path, monkeypatch):
|
||||
rows = comparison.load_results(TOOLS / "pimcomp/compare/results_comparison.csv")
|
||||
rows = [replace(row, energy=0.4) if row.model == "vgg8" and row.mode == "latency" else row for row in rows]
|
||||
captured = []
|
||||
monkeypatch.setattr(comparison, "save_figure", lambda fig, *args: captured.append(fig))
|
||||
comparison.draw_compact_summary(rows, tmp_path)
|
||||
fig = captured[0]
|
||||
try:
|
||||
fig.canvas.draw()
|
||||
assert all(ax.get_xlim()[0] == 0.25 for ax in fig.axes)
|
||||
assert "0.25" in [text.get_text() for text in fig.axes[0].get_xticklabels()]
|
||||
finally:
|
||||
comparison.plt.close(fig)
|
||||
|
||||
@@ -270,7 +484,7 @@ def test_compact_metrics_use_speedup_pipeline_selection(comparison, monkeypatch,
|
||||
captured.append((selected, field, basename, diverging))
|
||||
|
||||
monkeypatch.setattr(comparison, "load_results", lambda path: rows)
|
||||
for name in ("configure_style", "draw_validation", "draw_pipeline_effect", "draw_absolute"):
|
||||
for name in ("configure_style", "draw_validation", "draw_pipeline_effect", "draw_absolute", "draw_compact_summary"):
|
||||
monkeypatch.setattr(comparison, name, lambda *args, **kwargs: None)
|
||||
monkeypatch.setattr(comparison, "draw_metric", capture)
|
||||
monkeypatch.setattr(sys, "argv", ["generate_plots.py"])
|
||||
@@ -293,8 +507,8 @@ def test_compiler_palette_and_grayscale_patterns(comparison, tmp_path, monkeypat
|
||||
|
||||
assert comparison.COMPILER_STYLES == {
|
||||
"raptor": ("#B3D5CA", "#4B7C6A"), "pimcomp": ("#E1BC8B", "#856A44")}
|
||||
assert comparison.METRIC_STYLES == {
|
||||
"latency": ("#B3CCE3", "#456B90"), "throughput": ("#B3D5CA", "#4B7C6A")}
|
||||
assert comparison.ARCH_STYLES == {
|
||||
"arch-a": ("#B3CCE3", "#456B90", ""), "arch-b": ("#B3D5CA", "#4B7C6A", "//")}
|
||||
assert comparison.VALIDATION_STYLES["PASS"] == ("#B3D5CA", "#4B7C6A")
|
||||
assert comparison.TEXT_COLOR == "#222222"
|
||||
assert comparison.NETWORK_STYLES["yolo11n"] == ("#FF9494", "#C52020")
|
||||
@@ -304,7 +518,7 @@ def test_compiler_palette_and_grayscale_patterns(comparison, tmp_path, monkeypat
|
||||
for row in rows if row.model == "vgg8" and row.mode == "latency"]
|
||||
captured = []
|
||||
monkeypatch.setattr(comparison, "save_figure", lambda fig, *args: captured.append(fig))
|
||||
comparison.draw_metric(selected, "power", "Average power advantage (x)", "power", tmp_path, diverging=True)
|
||||
comparison.draw_metric(selected, "power", "Average power reduction (x)", "power", tmp_path, diverging=True)
|
||||
comparison.draw_absolute(selected, "latency", "Single-inference latency (ms)", "latency", tmp_path)
|
||||
try:
|
||||
for ax, compiler in zip(captured[0].axes, ("raptor", "pimcomp")):
|
||||
@@ -319,6 +533,44 @@ def test_compiler_palette_and_grayscale_patterns(comparison, tmp_path, monkeypat
|
||||
comparison.plt.close(fig)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("hatch", ["//", "\\\\", ".."])
|
||||
@pytest.mark.parametrize("width,height", [(20, 130), (160, 15), (20, 1)])
|
||||
@pytest.mark.parametrize("clipped", [False, True])
|
||||
def test_hatch_geometry_stays_inside_bar(comparison, monkeypatch, hatch, width, height, clipped):
|
||||
from matplotlib.backends.backend_agg import RendererAgg
|
||||
from matplotlib.path import Path as PlotPath
|
||||
from matplotlib.transforms import Affine2D, Bbox
|
||||
from plot_style import CenteredHatch
|
||||
|
||||
renderer = RendererAgg(400, 300, 100)
|
||||
gc = renderer.new_gc()
|
||||
gc.set_hatch(hatch)
|
||||
gc.set_hatch_color((0, 0, 0, 1))
|
||||
path = PlotPath.unit_rectangle()
|
||||
transform = Affine2D().scale(width, height).translate(50, 70)
|
||||
bar = transform.transform_path(path).get_extents()
|
||||
if clipped:
|
||||
gc.set_clip_rectangle(Bbox.from_extents(55, 70.25, 180, 180))
|
||||
bar = Bbox.intersection(bar, gc.get_clip_rectangle())
|
||||
drawn = []
|
||||
|
||||
def capture(context, geometry, affine, face=None):
|
||||
drawn.append((context.get_hatch(), affine.transform_path(geometry).get_extents()))
|
||||
|
||||
def capture_markers(context, marker, marker_transform, centers, affine, face=None):
|
||||
for x, y in affine.transform_path(centers).vertices:
|
||||
capture(context, marker, marker_transform + Affine2D().translate(x, y), face)
|
||||
|
||||
monkeypatch.setattr(renderer, "draw_path", capture)
|
||||
monkeypatch.setattr(renderer, "draw_markers", capture_markers)
|
||||
CenteredHatch().draw_path(renderer, gc, path, transform, (1, 1, 1, 1))
|
||||
assert len(drawn) > 1
|
||||
assert all(pattern is None for pattern, _ in drawn)
|
||||
for _, bounds in drawn[1:]:
|
||||
assert bar.x0 - 1e-8 <= bounds.x0 <= bounds.x1 <= bar.x1 + 1e-8
|
||||
assert bar.y0 - 1e-8 <= bounds.y0 <= bounds.y1 <= bar.y1 + 1e-8
|
||||
|
||||
|
||||
def test_tick_rotation_changes_only_overlapping_labels(comparison):
|
||||
fig, ax = comparison.plt.subplots(figsize=(4, 3), layout="constrained")
|
||||
try:
|
||||
@@ -341,6 +593,14 @@ def test_all_plot_terminology(comparison, monkeypatch):
|
||||
|
||||
def capture(fig, name, output):
|
||||
fig.canvas.draw()
|
||||
if "compact" in name:
|
||||
compact_text = "\n".join(text.get_text().lower() for text in fig.findobj(Text))
|
||||
assert "single-inference" not in compact_text
|
||||
assert "best-case throughput" not in compact_text
|
||||
if name == "performance_energy_compact":
|
||||
labels.update(text.get_text() for text in fig.findobj(Text))
|
||||
plotter.plt.close(fig)
|
||||
return
|
||||
if name != "raptor_pipeline_throughput":
|
||||
assert fig.axes[1].get_ylabel() == ""
|
||||
assert fig.axes[1].get_yticklabels() == []
|
||||
@@ -393,13 +653,14 @@ def test_all_plot_terminology(comparison, monkeypatch):
|
||||
plotter.main()
|
||||
text = "\n".join(labels)
|
||||
for old in ("Pimcomp", "ARCHITECTURE", "VGG8", "RESNET", "GOOGLENET", "YOLO11N",
|
||||
"Single inference", "Best case", "Latency, 1 stage", "Energy per inference advantage"):
|
||||
"Single inference", "Best case", "Latency, 1 stage", "Energy per inference advantage",
|
||||
"Energy-per-inference advantage", "Energy consumption reduction", "power advantage"):
|
||||
assert old not in text
|
||||
assert {"VGG-8", "ResNet-18", "ResNet-34", "GoogLeNet"} <= labels
|
||||
assert "Arch-A" in text and "Arch-B" in text
|
||||
assert {"Raptor", "PIMCOMP", "YOLO11n", "Single-inference", "2 stages", "4 stages", "8 stages",
|
||||
"Single-inference latency (ms)", "Throughput (inferences/s)", "Energy per inference (mJ)",
|
||||
"Energy-per-inference advantage (x)", "Derived as 1 / latency",
|
||||
"Energy reduction (x)", "Average power reduction (x)", "Derived as 1 / latency",
|
||||
"Throughput measurement"} <= labels
|
||||
|
||||
|
||||
@@ -432,7 +693,7 @@ def test_advantage_throughput_groups_touch(comparison, tmp_path, monkeypatch, di
|
||||
selected = [row for row in rows if row.model in ("vgg8", "resnet18")]
|
||||
captured = []
|
||||
monkeypatch.setattr(comparison, "save_figure", lambda fig, *args: captured.append(fig))
|
||||
comparison.draw_metric(selected, "energy", "Energy-per-inference advantage (x)", "energy", tmp_path,
|
||||
comparison.draw_metric(selected, "energy", "Energy reduction (x)", "energy", tmp_path,
|
||||
diverging=diverging)
|
||||
fig = captured[0]
|
||||
try:
|
||||
@@ -466,8 +727,10 @@ def test_grouped_absolute_plot_scheduling_labels(comparison, tmp_path, monkeypat
|
||||
comparison.plt.close(fig)
|
||||
|
||||
|
||||
def test_pipeline_plot_connects_derived_and_measured_rates(comparison, tmp_path, monkeypatch):
|
||||
@pytest.mark.parametrize("status", ["PASS", "FAIL", "SKIPPED", "NA"])
|
||||
def test_pipeline_plot_connects_derived_and_measured_rates(comparison, tmp_path, monkeypatch, status):
|
||||
rows = comparison.load_results(TOOLS / "pimcomp/compare/results_comparison.csv")
|
||||
rows = [replace(row, raptor_validation=status, pimcomp_validation=status) for row in rows]
|
||||
captured = []
|
||||
monkeypatch.setattr(comparison, "save_figure", lambda fig, *args: captured.append(fig))
|
||||
comparison.draw_pipeline_effect(rows, tmp_path)
|
||||
@@ -518,6 +781,7 @@ def test_plotter_exports_png_svg_and_pdf(comparison, tmp_path):
|
||||
for row in rows if row.model == "vgg8" and row.mode == "latency"]
|
||||
comparison.draw_absolute(selected, "latency", "Latency (ms)", "comparison-smoke", tmp_path)
|
||||
comparison.draw_validation(rows, tmp_path, compact=True)
|
||||
comparison.draw_compact_summary(rows, tmp_path)
|
||||
compact_names = []
|
||||
for metric in ("speedup", "energy", "power"):
|
||||
for suffix, diverging in (("comparison", False), ("diverging", True)):
|
||||
@@ -525,10 +789,182 @@ def test_plotter_exports_png_svg_and_pdf(comparison, tmp_path):
|
||||
compact_names.append(name)
|
||||
comparison.draw_metric(comparison.select_compact_speedups(rows), metric, "Advantage (x)",
|
||||
name, tmp_path, diverging=diverging, compact=True)
|
||||
for name in ("comparison-smoke", "validation_status_compact", *compact_names):
|
||||
for name in ("comparison-smoke", "validation_status_compact", "performance_energy_compact", *compact_names):
|
||||
assert (tmp_path / f"png/{name}.png").read_bytes().startswith(b"\x89PNG\r\n\x1a\n")
|
||||
svg = (tmp_path / f"svg/{name}.svg").read_text()
|
||||
assert "<svg" in svg
|
||||
assert (tmp_path / f"pdf/{name}.pdf").read_bytes().startswith(b"%PDF-")
|
||||
assert "N/A" in (tmp_path / "svg/comparison-smoke.svg").read_text()
|
||||
assert "P. INVALID" in (tmp_path / "svg/comparison-smoke.svg").read_text()
|
||||
|
||||
|
||||
def test_compact_summary_ratios_validity_and_layout(comparison, tmp_path, monkeypatch):
|
||||
from matplotlib.path import Path as PlotPath
|
||||
from matplotlib.transforms import Affine2D, Bbox
|
||||
|
||||
rows = comparison.load_results(TOOLS / "pimcomp/compare/results_comparison.csv")
|
||||
selected = comparison.select_compact_speedups([row for row in rows if row.model != "yolo11n"])
|
||||
lookup = {(row.model, row.arch, row.mode): row for row in selected}
|
||||
captured = []
|
||||
monkeypatch.setattr(comparison, "save_figure", lambda fig, name, path: captured.append((fig, name)))
|
||||
comparison.draw_compact_summary(rows, tmp_path)
|
||||
fig, name = captured[0]
|
||||
try:
|
||||
assert name == "performance_energy_compact"
|
||||
assert len(fig.axes) == 4
|
||||
fig.canvas.draw()
|
||||
renderer = fig.canvas.get_renderer()
|
||||
keys = [(model, arch) for model in comparison.MODELS[:-1] for arch in comparison.ARCHITECTURES]
|
||||
for ax, mode, field in zip(fig.axes, ("latency", "throughput", "latency", "throughput"),
|
||||
("latency_speedup", "throughput_speedup", "energy", "energy")):
|
||||
assert ax.get_xscale() == "linear"
|
||||
low, high = ax.get_xlim()
|
||||
assert low == 0.5 < 1 < high
|
||||
points = [line for line in ax.lines if line.get_marker() in ("o", "s", "^", "D")]
|
||||
point_keys = [(*key, mode) for key in keys]
|
||||
assert len(points) == len(point_keys)
|
||||
expected, invalid_flags = [], []
|
||||
for point, key in zip(points, point_keys):
|
||||
row = lookup[key]
|
||||
ratio = getattr(row, field)
|
||||
expected.append(ratio)
|
||||
assert list(point.get_xdata()) == pytest.approx([ratio])
|
||||
assert list(point.get_ydata()) == [0 if row.arch == "arch-a" else 2]
|
||||
assert point.get_marker() == {"vgg8": "o", "resnet18": "s", "resnet34": "^", "googlenet": "D"}[row.model]
|
||||
fill, border = comparison.NETWORK_STYLES[row.model]
|
||||
assert point.get_markerfacecolor() == fill
|
||||
assert point.get_markeredgecolor() == border
|
||||
invalid = row.mode == "throughput" or row.pimcomp_validation == "FAIL"
|
||||
invalid_flags.append(invalid)
|
||||
labels = [text for text in ax.texts if getattr(text, "xycoords", None) == "data"]
|
||||
groups = []
|
||||
for text, key, ratio, invalid in zip(labels, point_keys, expected, invalid_flags):
|
||||
assert text.get_text() == f"{ratio:.2f}×"
|
||||
assert text.get_math_fontfamily() == "cm"
|
||||
assert text.get_color() == comparison.TEXT_COLOR
|
||||
assert text.get_rotation() == 90
|
||||
assert text.arrow_patch is None
|
||||
assert abs(text.get_position()[0]) <= 8
|
||||
assert (text.get_position()[1] > 0) == (key[1] == "arch-a")
|
||||
children = [child for child in ax.texts if getattr(child, "xycoords", None) is text]
|
||||
assert len(children) == 1
|
||||
for dagger in children:
|
||||
assert dagger.get_text() == r"$\boldsymbol{\dagger}$"
|
||||
assert dagger.get_color() == comparison.DAGGER_COLOR == "#AA1C1C"
|
||||
assert dagger.get_fontsize() == 9
|
||||
assert dagger.get_fontsize() > text.get_fontsize()
|
||||
assert dagger.get_alpha() == int(invalid)
|
||||
groups.append([text, *children])
|
||||
for lane in (0, 2):
|
||||
offsets = [text.get_window_extent(renderer).y1 - ax.transData.transform(text.xy)[1]
|
||||
for text in labels if text.xy[1] == lane]
|
||||
assert max(offsets) == pytest.approx(min(offsets))
|
||||
assert len(labels) == 8
|
||||
assert list(ax.get_yticks()) == [0, 2]
|
||||
assert [text.get_text() for text in ax.get_yticklabels()] == ["Arch-A", "Arch-B"]
|
||||
assert ax.get_title() == ""
|
||||
assert ax.yaxis.label.get_rotation() == 90
|
||||
assert ax.yaxis.label._get_multialignment() == "center"
|
||||
assert ax.yaxis.label.get_fontsize() == 8.5
|
||||
assert len(ax.get_ylabel().splitlines()) <= 2
|
||||
assert ax.yaxis.label.get_window_extent(renderer).x1 < ax.get_window_extent(renderer).x0
|
||||
assert bool(ax.get_xticklabels()) == (ax is fig.axes[0])
|
||||
if ax is fig.axes[0]:
|
||||
assert "0.5" in [text.get_text() for text in ax.get_xticklabels()]
|
||||
assert "0" not in [text.get_text() for text in ax.get_xticklabels()]
|
||||
assert all(tick.label2.get_visible() and not tick.label1.get_visible()
|
||||
for tick in ax.xaxis.get_major_ticks())
|
||||
assert not any(text.get_text() in ("Raptor / PIMCOMP", "PIMCOMP / Raptor") for text in ax.texts)
|
||||
baseline = next(line for line in ax.lines if list(line.get_xdata()) == [1, 1])
|
||||
assert baseline.get_linestyle() == "--"
|
||||
bounds = ax.get_window_extent(renderer)
|
||||
paths = []
|
||||
for line in ax.lines:
|
||||
box = line.get_window_extent(renderer).padded(1)
|
||||
paths.append(PlotPath.unit_rectangle().transformed(
|
||||
Affine2D().scale(box.width, box.height).translate(box.x0, box.y0)))
|
||||
for group in groups:
|
||||
text = group[0]
|
||||
box = Bbox.union([part.get_window_extent(renderer) for part in group])
|
||||
assert bounds.x0 <= box.x0 <= box.x1 <= bounds.x1
|
||||
assert bounds.y0 <= box.y0 <= box.y1 <= bounds.y1
|
||||
path = PlotPath.unit_rectangle().transformed(
|
||||
Affine2D().scale(box.width, box.height).translate(box.x0, box.y0))
|
||||
assert not any(path.intersects_path(other, filled=True) for other in paths), text.get_text()
|
||||
paths.append(path)
|
||||
assert "YOLO11n" not in [text.get_text() for text in ax.get_yticklabels()]
|
||||
assert {text.get_position()[1] > 0 for ax in fig.axes for text in ax.texts
|
||||
if getattr(text, "xycoords", None) == "data"} == {False, True}
|
||||
assert all(ax.get_xlim() == fig.axes[0].get_xlim() for ax in fig.axes)
|
||||
assert [ax.get_ylabel() for ax in fig.axes] == [
|
||||
"Latency-mode\nspeedup", "Throughput-mode\nspeedup",
|
||||
"Latency-mode\nenergy reduction", "Throughput-mode\nenergy reduction"]
|
||||
assert fig.axes[0].xaxis.get_label_position() == "top"
|
||||
xlabel = fig.axes[0].xaxis.label.get_window_extent(renderer)
|
||||
legend = fig.legends[0].get_window_extent(renderer)
|
||||
assert fig.axes[0].get_window_extent(renderer).y1 < xlabel.y0 < xlabel.y1 < legend.y0
|
||||
tick_top = max(text.get_window_extent(renderer).y1 for text in fig.axes[0].get_xticklabels())
|
||||
assert 0 < xlabel.y0 - tick_top < renderer.points_to_pixels(12)
|
||||
axis_bounds = fig.axes[0].get_window_extent(renderer)
|
||||
assert (xlabel.x0 + xlabel.x1) / 2 == pytest.approx((axis_bounds.x0 + axis_bounds.x1) / 2)
|
||||
assert fig.axes[0].get_xlabel() == "Ratio (x)"
|
||||
assert {"← PIMCOMP better", "Raptor better →"} <= {text.get_text() for text in fig.axes[0].texts}
|
||||
left, right = [text.get_window_extent(renderer) for text in fig.axes[0].texts if not hasattr(text, "xy")]
|
||||
assert left.y0 == right.y0 == pytest.approx(xlabel.y0)
|
||||
assert left.x1 < xlabel.x0 < xlabel.x1 < right.x0
|
||||
legend_labels = {text.get_text() for text in fig.legends[0].get_texts() if text.get_visible()}
|
||||
assert legend_labels == {"VGG-8", "ResNet-18", "ResNet-34", "GoogLeNet", "PIMCOMP invalid"}
|
||||
for handle, text in zip(fig.legends[0].legend_handles, fig.legends[0].get_texts()):
|
||||
if text.get_text() in comparison.MODEL_NAMES.values():
|
||||
model = next(model for model, name in comparison.MODEL_NAMES.items() if name == text.get_text())
|
||||
assert handle.get_marker() == {"vgg8": "o", "resnet18": "s", "resnet34": "^", "googlenet": "D"}[model]
|
||||
assert handle.get_markerfacecolor() == comparison.NETWORK_STYLES[model][0]
|
||||
elif text.get_text() == "PIMCOMP invalid":
|
||||
assert handle.get_text() == r"$\boldsymbol{\dagger}$"
|
||||
assert handle.get_fontsize() == 9
|
||||
assert handle.get_math_fontfamily() == "cm"
|
||||
assert handle.get_color() == comparison.DAGGER_COLOR
|
||||
result_dagger = next(child for child in fig.axes[0].texts
|
||||
if getattr(getattr(child, "xycoords", None), "xycoords", None) == "data"
|
||||
and child.get_alpha() == 1)
|
||||
result_box = result_dagger.get_window_extent(renderer)
|
||||
legend_box = handle.get_window_extent(renderer)
|
||||
assert legend_box.width == pytest.approx(result_box.height)
|
||||
assert legend_box.height == pytest.approx(result_box.width)
|
||||
ratios = [line.get_xdata()[0] for ax in fig.axes for line in ax.lines
|
||||
if line.get_marker() in ("o", "s", "^", "D")]
|
||||
lower, upper = fig.axes[0].get_xlim()
|
||||
padding = (max(ratios) - min(ratios)) * 0.04
|
||||
assert lower == 0.5
|
||||
assert upper - max(ratios) == pytest.approx(padding)
|
||||
finally:
|
||||
comparison.plt.close(fig)
|
||||
|
||||
|
||||
def test_compact_summary_rejects_missing_ratio(comparison, tmp_path):
|
||||
rows = comparison.load_results(TOOLS / "pimcomp/compare/results_comparison.csv")
|
||||
rows = [replace(row, energy=math.nan) if row.mode == "throughput" and row.model == "vgg8"
|
||||
else row for row in rows]
|
||||
with pytest.raises(ValueError, match="requires a positive energy ratio"):
|
||||
comparison.draw_compact_summary(rows, tmp_path)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("compiler,status,label", [("pimcomp", "FAIL", r"$\boldsymbol{\dagger}$"), ("raptor", "FAIL", "R. INVALID"),
|
||||
("pimcomp", "SKIPPED", "NOT VALIDATED"), ("pimcomp", "NA", "NOT VALIDATED")])
|
||||
def test_compact_summary_validation_labels(comparison, tmp_path, monkeypatch, compiler, status, label):
|
||||
rows = comparison.load_results(TOOLS / "pimcomp/compare/results_comparison.csv")
|
||||
rows = [replace(row, **{f"{compiler}_validation": status})
|
||||
if row.model == "vgg8" and row.arch == "arch-a" and row.mode == "latency" else row for row in rows]
|
||||
captured = []
|
||||
monkeypatch.setattr(comparison, "save_figure", lambda fig, *args: captured.append(fig))
|
||||
comparison.draw_compact_summary(rows, tmp_path)
|
||||
fig = captured[0]
|
||||
try:
|
||||
ax = fig.axes[0]
|
||||
if compiler == "pimcomp" and status == "FAIL":
|
||||
assert any(label in text.get_text() and text.get_color() == comparison.DAGGER_COLOR for text in ax.texts)
|
||||
else:
|
||||
assert label in ax.texts[0].get_text()
|
||||
assert ax.texts[0].get_color() == comparison.TEXT_COLOR
|
||||
finally:
|
||||
comparison.plt.close(fig)
|
||||
|
||||
Reference in New Issue
Block a user