better compile times for throughput scheduling
Validate Operations / validate-operations (push) Has been cancelled

fix failing no-plan throughput ablation runs
update results
This commit is contained in:
NiccoloN
2026-10-28 16:34:48 +01:00
parent 3c7679594b
commit 5f2db946f0
28 changed files with 909 additions and 197 deletions
+43 -1
View File
@@ -511,7 +511,9 @@ def test_comparison_uses_same_model_for_reference_and_execution(tmp_path, monkey
def test_yolo_uses_full_model_without_moving_comparison_artifacts():
assert suite.MODELS["yolo11n"] == suite.REPO / "validation/networks/yolo11n/depth_51/yolo11n_depth_51.onnx"
assert suite.MODELS["yolo11n"] == suite.SUITE / "yolo11n/yolo11n.onnx"
assert suite.MODELS["yolo11n"].is_symlink()
assert suite.MODELS["yolo11n"].resolve() == suite.REPO / "validation/networks/yolo11n/depth_51/yolo11n_depth_51.onnx"
assert suite.model_dir(None, "yolo11n") == suite.SUITE / "yolo11n"
assert suite.common_dir(None, "yolo11n") == suite.SUITE / "yolo11n/artifacts/common-full-model"
@@ -756,3 +758,43 @@ def test_ablation_aggregate_keeps_pass_with_zero_performance(tmp_path):
assert rows[0]["functional_validation"] == "PASS"
assert rows[0]["baseline_functional_validation"] == "PASS"
assert all(rows[0][f"{metric}_gain_percent"] == "NA" for _, metric in ablation.PERCENTAGE_FIELDS)
def test_completed_no_sync_ablation_preserves_numerical_failure():
report = {"pimsim_mode": "throughput", "raptor_extra_args": ["--pim-disable-synchronization"],
"raptor_validation": {"status": "done", "passed": False, "max_diffs": {"output": 1}},
"raptor_performance": {"throughput": 100, "output_count": 8},
"steps": [{"name": "Functional validation Raptor", "status": "passed"}],
"failures": []}
assert suite.completed_synchronization_ablation(report)
assert not suite.comparison_passed(report)
assert suite.functional_validation_status(report["raptor_validation"]) == "FAIL"
for field, value in (("pimsim_mode", "latency"), ("raptor_extra_args", []),
("raptor_validation", {"status": "failed", "passed": False}),
("raptor_validation", {"status": "done", "passed": False}),
("raptor_performance", {"error": "SIGSEGV", "throughput": 100}),
("raptor_performance", {"throughput": 100, "output_count": 0}),
("steps", [{"status": "failed"}]),
("failures", [{"stage": "Compile Raptor Pim", "error": "failed"}])):
assert not suite.completed_synchronization_ablation({**report, field: value})
@pytest.mark.parametrize("process_failed", [False, True])
@pytest.mark.parametrize("variant,expected", [("no-sync", 0), ("no-plan", 1), ("baseline", 1)])
def test_ablation_runner_distinguishes_invalid_outputs_from_failed_execution(tmp_path, monkeypatch, variant, expected, process_failed):
report = {"pimsim_mode": "throughput", "raptor_extra_args": ["--pim-disable-synchronization"],
"raptor_validation": {"status": "done", "passed": False, "max_diffs": {"output": 1}},
"raptor_performance": {"throughput": 100, "output_count": 8},
"steps": [{"status": "passed"}], "failures": []}
output = suite.result_dir(tmp_path, "vgg8", "arch-a", "throughput", 4, variant) / "pimcomp/comparison_report.json"
output.parent.mkdir(parents=True)
output.write_text(json.dumps(report))
csv = tmp_path / "results.csv"
csv.write_text("summary\n")
monkeypatch.setattr(suite, "run", lambda *a, **k: 0)
monkeypatch.setattr(suite, "run_comparison_jobs", lambda *a: (["arch-a/vgg8/throughput/pipeline4"] if process_failed else [], 1))
monkeypatch.setattr(suite, "write_results_csv", lambda *a: csv)
monkeypatch.setattr(sys, "argv", ["run_pimcomp_models.py", "--out-dir", str(tmp_path),
"--models", "vgg8", "--archs", "arch-a", "--mode", "throughput", "--pipeline", "4",
"--raptor-only", "--ablation-variant", variant, "--jobs", "1"])
assert suite.main() == (1 if process_failed else expected)