force selected pipeline stages in throughput mode
Validate Operations / validate-operations (push) Has been cancelled

increase throughput scheduler budget
add experimental throughput auto mode
This commit is contained in:
NiccoloN
2026-11-02 12:08:33 +01:00
parent 4168b588fa
commit 4a1fe710b2
35 changed files with 2047 additions and 398 deletions
+71
View File
@@ -20,9 +20,80 @@ sys.path.insert(0, str(TOOLS / "pimcomp/compare"))
import run_ablation as ablation
import run_pimcomp_models as suite
import compare_raptor_pimcomp_model as comparator
from raptor_validation.raptor import parse_pipeline_depth, pipeline_sort_key, read_selected_pipeline_depth
from raptor_validation.validate_one import validate_outputs
def test_default_experiment_depths_exclude_auto():
expected = [("latency", 1), ("throughput", 2), ("throughput", 4), ("throughput", 8)]
assert [(mode, depth) for mode, depth, _ in suite.COMPARISONS] == expected
cases = ablation.runner_argument_sets(["--mode", "latency", "throughput"])
assert [ablation.case_key(arguments) for _, arguments in cases] == [
(mode, str(depth)) for mode, depth in expected]
assert all("auto" not in arguments for _, arguments in ablation.runner_argument_sets([]))
def test_operation_validation_forwards_requested_depth(tmp_path, monkeypatch):
import importlib.util
spec = importlib.util.spec_from_file_location("operation_validation", TOOLS.parents[1] / "validate.py")
operation_validation = importlib.util.module_from_spec(spec)
spec.loader.exec_module(operation_validation)
model = tmp_path / "model.onnx"
model.touch()
monkeypatch.setattr(operation_validation, "discover_onnx_files", lambda _: [model])
monkeypatch.setattr(sys, "argv", ["validate.py", "--operations-dir", str(tmp_path),
"--raptor-path", str(tmp_path / "compiler"), "--onnx-include-dir", str(tmp_path),
"--skip-non-functional-simulation", "--jobs", "1", "--throughput-pipeline", "8"])
class JobReached(Exception):
pass
def check_job(job):
assert job[1]["validation_kwargs"]["throughput_pipeline"] == 8
raise JobReached
monkeypatch.setattr(operation_validation, "run_validation_job", check_job)
with pytest.raises(JobReached):
operation_validation.main()
@pytest.mark.parametrize("depth", ["auto", "3"])
def test_comparison_and_ablation_accept_depth_domain(tmp_path, monkeypatch, depth):
commands = []
monkeypatch.setattr(suite, "run", lambda command, **kwargs: commands.append(command) or 0)
monkeypatch.setattr(sys, "argv", ["run_pimcomp_models.py", "--out-dir", str(tmp_path),
"--models", "vgg8", "--archs", "arch-b", "--mode", "throughput", "--pipeline", depth,
"--raptor-only", "--dry-run"])
assert suite.main() == 0
assert len(commands) == 2
assert f"--raptor-extra-arg=--pipeline={depth}" in commands[-1]
assert f"pipeline{depth}" in commands[-1][commands[-1].index("--out-dir") + 1]
cases = ablation.runner_argument_sets(["--mode", "throughput", "--pipeline", depth])
assert len(cases) == 1 and cases[0][0] == f"throughput/pipeline{depth}"
assert cases[0][1][-1] == depth
def test_pipeline_depth_parser():
for invalid in ["0", "-1", "unknown", "2.5"]:
with pytest.raises(argparse.ArgumentTypeError):
parse_pipeline_depth(invalid)
assert parse_pipeline_depth("1") == 1
assert parse_pipeline_depth("auto") == "auto"
assert sorted(["auto", "8", "1", "3"], key=pipeline_sort_key) == ["1", "3", "8", "auto"]
def test_selected_pipeline_report(tmp_path):
assert read_selected_pipeline_depth(None) is None
assert read_selected_pipeline_depth(tmp_path / "pim") is None
(tmp_path / "reports").mkdir()
report = tmp_path / "reports/pipeline_selection.json"
report.write_text(json.dumps({"requested_depth": "auto", "selected_depth": 3}))
assert read_selected_pipeline_depth(tmp_path / "pim") == 3
report.write_text(json.dumps({"requested_depth": 8}))
assert read_selected_pipeline_depth(tmp_path / "pim") is None
@pytest.mark.parametrize("mode", ["latency", "throughput"])
def test_comparison_report_energy_is_per_inference(tmp_path, mode):
report = tmp_path / "comparison_report.md"