better compile times for throughput scheduling
Validate Operations / validate-operations (push) Has been cancelled
Validate Operations / validate-operations (push) Has been cancelled
fix failing no-plan throughput ablation runs update results
This commit is contained in:
@@ -511,7 +511,9 @@ def test_comparison_uses_same_model_for_reference_and_execution(tmp_path, monkey
|
||||
|
||||
|
||||
def test_yolo_uses_full_model_without_moving_comparison_artifacts():
|
||||
assert suite.MODELS["yolo11n"] == suite.REPO / "validation/networks/yolo11n/depth_51/yolo11n_depth_51.onnx"
|
||||
assert suite.MODELS["yolo11n"] == suite.SUITE / "yolo11n/yolo11n.onnx"
|
||||
assert suite.MODELS["yolo11n"].is_symlink()
|
||||
assert suite.MODELS["yolo11n"].resolve() == suite.REPO / "validation/networks/yolo11n/depth_51/yolo11n_depth_51.onnx"
|
||||
assert suite.model_dir(None, "yolo11n") == suite.SUITE / "yolo11n"
|
||||
assert suite.common_dir(None, "yolo11n") == suite.SUITE / "yolo11n/artifacts/common-full-model"
|
||||
|
||||
@@ -756,3 +758,43 @@ def test_ablation_aggregate_keeps_pass_with_zero_performance(tmp_path):
|
||||
assert rows[0]["functional_validation"] == "PASS"
|
||||
assert rows[0]["baseline_functional_validation"] == "PASS"
|
||||
assert all(rows[0][f"{metric}_gain_percent"] == "NA" for _, metric in ablation.PERCENTAGE_FIELDS)
|
||||
|
||||
|
||||
def test_completed_no_sync_ablation_preserves_numerical_failure():
|
||||
report = {"pimsim_mode": "throughput", "raptor_extra_args": ["--pim-disable-synchronization"],
|
||||
"raptor_validation": {"status": "done", "passed": False, "max_diffs": {"output": 1}},
|
||||
"raptor_performance": {"throughput": 100, "output_count": 8},
|
||||
"steps": [{"name": "Functional validation Raptor", "status": "passed"}],
|
||||
"failures": []}
|
||||
assert suite.completed_synchronization_ablation(report)
|
||||
assert not suite.comparison_passed(report)
|
||||
assert suite.functional_validation_status(report["raptor_validation"]) == "FAIL"
|
||||
for field, value in (("pimsim_mode", "latency"), ("raptor_extra_args", []),
|
||||
("raptor_validation", {"status": "failed", "passed": False}),
|
||||
("raptor_validation", {"status": "done", "passed": False}),
|
||||
("raptor_performance", {"error": "SIGSEGV", "throughput": 100}),
|
||||
("raptor_performance", {"throughput": 100, "output_count": 0}),
|
||||
("steps", [{"status": "failed"}]),
|
||||
("failures", [{"stage": "Compile Raptor Pim", "error": "failed"}])):
|
||||
assert not suite.completed_synchronization_ablation({**report, field: value})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("process_failed", [False, True])
|
||||
@pytest.mark.parametrize("variant,expected", [("no-sync", 0), ("no-plan", 1), ("baseline", 1)])
|
||||
def test_ablation_runner_distinguishes_invalid_outputs_from_failed_execution(tmp_path, monkeypatch, variant, expected, process_failed):
|
||||
report = {"pimsim_mode": "throughput", "raptor_extra_args": ["--pim-disable-synchronization"],
|
||||
"raptor_validation": {"status": "done", "passed": False, "max_diffs": {"output": 1}},
|
||||
"raptor_performance": {"throughput": 100, "output_count": 8},
|
||||
"steps": [{"status": "passed"}], "failures": []}
|
||||
output = suite.result_dir(tmp_path, "vgg8", "arch-a", "throughput", 4, variant) / "pimcomp/comparison_report.json"
|
||||
output.parent.mkdir(parents=True)
|
||||
output.write_text(json.dumps(report))
|
||||
csv = tmp_path / "results.csv"
|
||||
csv.write_text("summary\n")
|
||||
monkeypatch.setattr(suite, "run", lambda *a, **k: 0)
|
||||
monkeypatch.setattr(suite, "run_comparison_jobs", lambda *a: (["arch-a/vgg8/throughput/pipeline4"] if process_failed else [], 1))
|
||||
monkeypatch.setattr(suite, "write_results_csv", lambda *a: csv)
|
||||
monkeypatch.setattr(sys, "argv", ["run_pimcomp_models.py", "--out-dir", str(tmp_path),
|
||||
"--models", "vgg8", "--archs", "arch-a", "--mode", "throughput", "--pipeline", "4",
|
||||
"--raptor-only", "--ablation-variant", variant, "--jobs", "1"])
|
||||
assert suite.main() == (1 if process_failed else expected)
|
||||
|
||||
Reference in New Issue
Block a user