force selected pipeline stages in throughput mode
Validate Operations / validate-operations (push) Has been cancelled
Validate Operations / validate-operations (push) Has been cancelled
increase throughput scheduler budget add experimental throughput auto mode
This commit is contained in:
@@ -20,9 +20,80 @@ sys.path.insert(0, str(TOOLS / "pimcomp/compare"))
|
||||
import run_ablation as ablation
|
||||
import run_pimcomp_models as suite
|
||||
import compare_raptor_pimcomp_model as comparator
|
||||
from raptor_validation.raptor import parse_pipeline_depth, pipeline_sort_key, read_selected_pipeline_depth
|
||||
from raptor_validation.validate_one import validate_outputs
|
||||
|
||||
|
||||
def test_default_experiment_depths_exclude_auto():
|
||||
expected = [("latency", 1), ("throughput", 2), ("throughput", 4), ("throughput", 8)]
|
||||
assert [(mode, depth) for mode, depth, _ in suite.COMPARISONS] == expected
|
||||
cases = ablation.runner_argument_sets(["--mode", "latency", "throughput"])
|
||||
assert [ablation.case_key(arguments) for _, arguments in cases] == [
|
||||
(mode, str(depth)) for mode, depth in expected]
|
||||
assert all("auto" not in arguments for _, arguments in ablation.runner_argument_sets([]))
|
||||
|
||||
|
||||
def test_operation_validation_forwards_requested_depth(tmp_path, monkeypatch):
|
||||
import importlib.util
|
||||
|
||||
spec = importlib.util.spec_from_file_location("operation_validation", TOOLS.parents[1] / "validate.py")
|
||||
operation_validation = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(operation_validation)
|
||||
model = tmp_path / "model.onnx"
|
||||
model.touch()
|
||||
monkeypatch.setattr(operation_validation, "discover_onnx_files", lambda _: [model])
|
||||
monkeypatch.setattr(sys, "argv", ["validate.py", "--operations-dir", str(tmp_path),
|
||||
"--raptor-path", str(tmp_path / "compiler"), "--onnx-include-dir", str(tmp_path),
|
||||
"--skip-non-functional-simulation", "--jobs", "1", "--throughput-pipeline", "8"])
|
||||
|
||||
class JobReached(Exception):
|
||||
pass
|
||||
|
||||
def check_job(job):
|
||||
assert job[1]["validation_kwargs"]["throughput_pipeline"] == 8
|
||||
raise JobReached
|
||||
|
||||
monkeypatch.setattr(operation_validation, "run_validation_job", check_job)
|
||||
with pytest.raises(JobReached):
|
||||
operation_validation.main()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("depth", ["auto", "3"])
|
||||
def test_comparison_and_ablation_accept_depth_domain(tmp_path, monkeypatch, depth):
|
||||
commands = []
|
||||
monkeypatch.setattr(suite, "run", lambda command, **kwargs: commands.append(command) or 0)
|
||||
monkeypatch.setattr(sys, "argv", ["run_pimcomp_models.py", "--out-dir", str(tmp_path),
|
||||
"--models", "vgg8", "--archs", "arch-b", "--mode", "throughput", "--pipeline", depth,
|
||||
"--raptor-only", "--dry-run"])
|
||||
assert suite.main() == 0
|
||||
assert len(commands) == 2
|
||||
assert f"--raptor-extra-arg=--pipeline={depth}" in commands[-1]
|
||||
assert f"pipeline{depth}" in commands[-1][commands[-1].index("--out-dir") + 1]
|
||||
cases = ablation.runner_argument_sets(["--mode", "throughput", "--pipeline", depth])
|
||||
assert len(cases) == 1 and cases[0][0] == f"throughput/pipeline{depth}"
|
||||
assert cases[0][1][-1] == depth
|
||||
|
||||
|
||||
def test_pipeline_depth_parser():
|
||||
for invalid in ["0", "-1", "unknown", "2.5"]:
|
||||
with pytest.raises(argparse.ArgumentTypeError):
|
||||
parse_pipeline_depth(invalid)
|
||||
assert parse_pipeline_depth("1") == 1
|
||||
assert parse_pipeline_depth("auto") == "auto"
|
||||
assert sorted(["auto", "8", "1", "3"], key=pipeline_sort_key) == ["1", "3", "8", "auto"]
|
||||
|
||||
|
||||
def test_selected_pipeline_report(tmp_path):
|
||||
assert read_selected_pipeline_depth(None) is None
|
||||
assert read_selected_pipeline_depth(tmp_path / "pim") is None
|
||||
(tmp_path / "reports").mkdir()
|
||||
report = tmp_path / "reports/pipeline_selection.json"
|
||||
report.write_text(json.dumps({"requested_depth": "auto", "selected_depth": 3}))
|
||||
assert read_selected_pipeline_depth(tmp_path / "pim") == 3
|
||||
report.write_text(json.dumps({"requested_depth": 8}))
|
||||
assert read_selected_pipeline_depth(tmp_path / "pim") is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", ["latency", "throughput"])
|
||||
def test_comparison_report_energy_is_per_inference(tmp_path, mode):
|
||||
report = tmp_path / "comparison_report.md"
|
||||
|
||||
Reference in New Issue
Block a user