force selected pipeline stages in throughput mode
Validate Operations / validate-operations (push) Has been cancelled

increase throughput scheduler budget
add experimental throughput auto mode
This commit is contained in:
NiccoloN
2026-11-02 12:08:33 +01:00
parent 4168b588fa
commit 4a1fe710b2
35 changed files with 2047 additions and 398 deletions
+6
View File
@@ -112,6 +112,12 @@ The regular validator uses these paths beside each model:
| `artifacts/raptor/throughput/pim/` | Pipeline-4 throughput artifacts. |
| `artifacts/simulation/{latency,throughput}/` | Functional simulator outputs. |
`--throughput-pipeline=N` selects an exact throughput depth; `auto` requests
automatic depth selection. The default remains 4. Compiler logs report the
selected depth when a selection report is available. Small operation graphs can
reject exact depth 4. Use `--throughput-pipeline=auto` to test their feasible
throughput depth without weakening explicit-depth constraints.
The [comparison runner](tools/pim/pimcomp/compare/README.md) has separate
architecture/pipeline directories and shared `artifacts/common/` references.
+32
View File
@@ -1,3 +1,5 @@
import argparse
import json
import re
import shlex
import shutil
@@ -20,6 +22,29 @@ PIM_PASS_LABEL_BY_SUFFIX = dict(PIM_PASS_LABELS)
TIMING_LINE_RE = re.compile(r"^\s*([0-9]+\.[0-9]+)\s+\(\s*[0-9.]+%\)\s+(.+?)\s*$")
def parse_pipeline_depth(value):
if value == "auto":
return value
try:
depth = int(value)
except ValueError:
raise argparse.ArgumentTypeError("pipeline must be a positive integer or auto") from None
if depth < 1:
raise argparse.ArgumentTypeError("pipeline must be a positive integer or auto")
return depth
def pipeline_sort_key(value):
return (str(value) == "auto", 0 if str(value) == "auto" else int(value))
def read_selected_pipeline_depth(pim_dir):
if pim_dir is None:
return None
report = Path(pim_dir).parent / "reports/pipeline_selection.json"
return json.loads(report.read_text()).get("selected_depth") if report.exists() else None
def _parse_pim_pass_timings(output_text):
pass_timings = {}
for line in output_text.splitlines():
@@ -79,6 +104,13 @@ def compile_with_raptor(network_path, raptor_onnx_path: Path, output_base: Path,
capture_output=True,
timeout_sec=timeout_sec,
)
selected_depth = read_selected_pipeline_depth(pim_dir)
if selected_depth is not None:
message = f"Selected pipeline depth: {selected_depth}"
if reporter is not None:
reporter.log(message)
else:
print(message)
if reporter is None:
print(Fore.GREEN + "Raptor execution successful" + Style.RESET_ALL)
return _parse_pim_pass_timings(output_text)
+71
View File
@@ -20,9 +20,80 @@ sys.path.insert(0, str(TOOLS / "pimcomp/compare"))
import run_ablation as ablation
import run_pimcomp_models as suite
import compare_raptor_pimcomp_model as comparator
from raptor_validation.raptor import parse_pipeline_depth, pipeline_sort_key, read_selected_pipeline_depth
from raptor_validation.validate_one import validate_outputs
def test_default_experiment_depths_exclude_auto():
expected = [("latency", 1), ("throughput", 2), ("throughput", 4), ("throughput", 8)]
assert [(mode, depth) for mode, depth, _ in suite.COMPARISONS] == expected
cases = ablation.runner_argument_sets(["--mode", "latency", "throughput"])
assert [ablation.case_key(arguments) for _, arguments in cases] == [
(mode, str(depth)) for mode, depth in expected]
assert all("auto" not in arguments for _, arguments in ablation.runner_argument_sets([]))
def test_operation_validation_forwards_requested_depth(tmp_path, monkeypatch):
import importlib.util
spec = importlib.util.spec_from_file_location("operation_validation", TOOLS.parents[1] / "validate.py")
operation_validation = importlib.util.module_from_spec(spec)
spec.loader.exec_module(operation_validation)
model = tmp_path / "model.onnx"
model.touch()
monkeypatch.setattr(operation_validation, "discover_onnx_files", lambda _: [model])
monkeypatch.setattr(sys, "argv", ["validate.py", "--operations-dir", str(tmp_path),
"--raptor-path", str(tmp_path / "compiler"), "--onnx-include-dir", str(tmp_path),
"--skip-non-functional-simulation", "--jobs", "1", "--throughput-pipeline", "8"])
class JobReached(Exception):
pass
def check_job(job):
assert job[1]["validation_kwargs"]["throughput_pipeline"] == 8
raise JobReached
monkeypatch.setattr(operation_validation, "run_validation_job", check_job)
with pytest.raises(JobReached):
operation_validation.main()
@pytest.mark.parametrize("depth", ["auto", "3"])
def test_comparison_and_ablation_accept_depth_domain(tmp_path, monkeypatch, depth):
commands = []
monkeypatch.setattr(suite, "run", lambda command, **kwargs: commands.append(command) or 0)
monkeypatch.setattr(sys, "argv", ["run_pimcomp_models.py", "--out-dir", str(tmp_path),
"--models", "vgg8", "--archs", "arch-b", "--mode", "throughput", "--pipeline", depth,
"--raptor-only", "--dry-run"])
assert suite.main() == 0
assert len(commands) == 2
assert f"--raptor-extra-arg=--pipeline={depth}" in commands[-1]
assert f"pipeline{depth}" in commands[-1][commands[-1].index("--out-dir") + 1]
cases = ablation.runner_argument_sets(["--mode", "throughput", "--pipeline", depth])
assert len(cases) == 1 and cases[0][0] == f"throughput/pipeline{depth}"
assert cases[0][1][-1] == depth
def test_pipeline_depth_parser():
for invalid in ["0", "-1", "unknown", "2.5"]:
with pytest.raises(argparse.ArgumentTypeError):
parse_pipeline_depth(invalid)
assert parse_pipeline_depth("1") == 1
assert parse_pipeline_depth("auto") == "auto"
assert sorted(["auto", "8", "1", "3"], key=pipeline_sort_key) == ["1", "3", "8", "auto"]
def test_selected_pipeline_report(tmp_path):
assert read_selected_pipeline_depth(None) is None
assert read_selected_pipeline_depth(tmp_path / "pim") is None
(tmp_path / "reports").mkdir()
report = tmp_path / "reports/pipeline_selection.json"
report.write_text(json.dumps({"requested_depth": "auto", "selected_depth": 3}))
assert read_selected_pipeline_depth(tmp_path / "pim") == 3
report.write_text(json.dumps({"requested_depth": 8}))
assert read_selected_pipeline_depth(tmp_path / "pim") is None
@pytest.mark.parametrize("mode", ["latency", "throughput"])
def test_comparison_report_energy_is_per_inference(tmp_path, mode):
report = tmp_path / "comparison_report.md"
+2
View File
@@ -31,6 +31,8 @@ Other [suite-runner options](../pimcomp/compare/README.md), including `--models`
`--archs`, and `--no-fast`, are forwarded. Supplying `--mode` or `--pipeline`
replaces the default latency/pipeline-4 case set. Mode-only selection runs all
suite pipelines for that mode. Pipeline-only selection uses its matching mode.
`--pipeline auto` requests automatic throughput depth selection; positive numeric
depths retain their exact-stage meaning.
`--only` is rejected because
the wrapper always runs Raptor only. Ctrl+C stops the active variant's workers.
@@ -25,6 +25,7 @@ from raptor_validation.pimcomp_models import (
add_models_argument,
)
from raptor_validation.artifacts import remove_lock_files
from raptor_validation.raptor import parse_pipeline_depth, pipeline_sort_key
@dataclass(frozen=True)
@@ -102,10 +103,12 @@ def runner_argument_sets(forwarded: list[str]) -> tuple[tuple[str, list[str]], .
)
parser = argparse.ArgumentParser(add_help=False, allow_abbrev=False)
parser.add_argument("--mode", nargs="+", choices=("latency", "throughput"))
parser.add_argument("--pipeline", type=int, choices=(1, 2, 4, 8))
parser.add_argument("--pipeline", type=parse_pipeline_depth)
args, remaining = parser.parse_known_args(forwarded)
cases = [(mode, pipeline) for mode, pipeline in
(("latency", 1), ("throughput", 2), ("throughput", 4), ("throughput", 8))
requested = (("latency", 1), ("throughput", 2), ("throughput", 4), ("throughput", 8)) if args.pipeline is None else (
("latency", 1) if args.pipeline == 1 else ("throughput", args.pipeline),
)
cases = [(mode, pipeline) for mode, pipeline in requested
if (args.mode is None or mode in args.mode)
and (args.pipeline is None or pipeline == args.pipeline)]
if not cases:
@@ -268,7 +271,7 @@ def aggregate_results(
row["arch"],
row["model"],
row["mode"],
int(row["raptor_pipeline"]),
pipeline_sort_key(row["raptor_pipeline"]),
row["comparison"],
)
)
@@ -12,12 +12,15 @@ built release Raptor, Pimcomp, and `pimsim-nn`.
Defaults: all five models, Arch-A/B, latency pipeline 1, throughput pipelines
2/4/8, four workers, and a functional throughput batch of 64.
Use `--help` for the complete option reference.
Use `--help` for the complete option reference. Numeric depths greater than one
require exactly that many active compute stages. `--pipeline auto` requests
throughput depth selection. Reports include `raptor_selected_pipeline`; the
requested depth remains in the existing pipeline field.
| Option | Purpose |
|---|---|
| `--models MODEL [...]`, `--archs ARCH [...]` | Select models and hardware profiles. |
| `--mode {latency,throughput} [...]`, `--pipeline {1,2,4,8}` | Select experiments. |
| `--mode {latency,throughput} [...]`, `--pipeline N\|auto` | Select experiments. |
| `--out-dir PATH` | Suite root; default: `validation/networks/pimcomp_models`. |
| `--common-dir PATH` | Override the shared reference-artifact root. |
| `--only {raptor,pimcomp}` | Rerun one compiler and reuse the other's existing report. |
@@ -44,7 +44,7 @@ from raptor_validation.onnx_utils import (
write_input_batch_csv,
write_inputs_to_memory_bin,
)
from raptor_validation.raptor import compile_with_raptor
from raptor_validation.raptor import compile_with_raptor, read_selected_pipeline_depth
from raptor_validation.output_comparison import VALIDATION_POLICY, compare_output_arrays
from raptor_validation.pimsim_nn import (
export_raptor_pimsim_artifact,
@@ -2160,6 +2160,7 @@ def main():
"raptor_reuse_context": experiment_context(args, model_path, functional_model_path, hardware, "raptor"),
"pimcomp_config": str(args.pimcomp_config),
"raptor_extra_args": args.raptor_extra_arg,
"raptor_selected_pipeline": read_selected_pipeline_depth(raptor_pim_dir),
"reused_raptor_report": optional_path(args.reuse_raptor_report.resolve()) if reuse_raptor else None,
"reused_pimcomp_report": optional_path(reused_pimcomp_report_path) if reuse_pimcomp else None,
"failures": failures,
@@ -30,6 +30,7 @@ from raptor_validation.pimcomp_models import (
from raptor_validation.artifacts import artifacts_dir, remove_lock_files
from raptor_validation.pimsim_nn import parse_pimsim_nn_metrics
from raptor_validation.output_comparison import VALIDATION_POLICY
from raptor_validation.raptor import parse_pipeline_depth, pipeline_sort_key
from raptor_validation.validate_one import STAGE_COLORS
PIMCOMP_SOURCE = REPO / "third_party/PIMCOMP-NN"
@@ -56,7 +57,7 @@ class ComparisonSpec:
common_dir: Path
config: Path
mode: str
pipeline: int
pipeline: int | str
pimcomp_pipeline: str
shared_pimcomp_dir: Path
anchor: bool
@@ -71,7 +72,7 @@ def result_dir(
name: str,
arch: str,
mode: str,
pipeline: int,
pipeline: int | str,
ablation_variant: str | None = None,
) -> Path:
base = artifacts_dir(model_dir(root, name))
@@ -243,7 +244,7 @@ def write_results_csv(
row["arch"],
model_order[row["model"]],
mode_order[row["mode"]],
int(row["raptor_pipeline"]),
pipeline_sort_key(row["raptor_pipeline"]),
)
)
output.parent.mkdir(parents=True, exist_ok=True)
@@ -378,7 +379,7 @@ def comparison_command(
common_dir: Path,
config: Path,
mode: str,
pipeline: int,
pipeline: int | str,
pimcomp_pipeline: str,
pimsim_time_ms: int,
batch_size: int,
@@ -610,9 +611,8 @@ def main() -> int:
)
parser.add_argument(
"--pipeline",
type=int,
choices=(1, 2, 4, 8),
help="Run only this Raptor pipeline within the selected mode.",
type=parse_pipeline_depth,
help="Run this positive Raptor pipeline depth, or auto, within the selected mode.",
)
parser.add_argument(
"--pimsim-time-ms",
@@ -684,11 +684,15 @@ def main() -> int:
parser.error("--batch-size must be positive")
if args.timeout_seconds < 0:
parser.error("--timeout-seconds must be non-negative")
comparisons_by_arch: dict[str, tuple[tuple[str, int, str], ...]] = {}
comparisons_by_arch: dict[str, tuple[tuple[str, int | str, str], ...]] = {}
configs_by_arch: dict[str, dict[str, Path]] = {}
for arch in args.archs:
requested = COMPARISONS if args.pipeline is None else (
("latency", 1, "element") if args.pipeline == 1
else ("throughput", args.pipeline, "batch"),
)
comparisons = tuple(
comparison for comparison in COMPARISONS
comparison for comparison in requested
if comparison[0] in args.mode
and (args.pipeline is None or comparison[1] == args.pipeline)
)
+4 -2
View File
@@ -30,7 +30,7 @@ from raptor_validation.validate_one import (
pimcomp_compatibility_errors,
validate_network,
)
from raptor_validation.raptor import PIM_PASS_LABELS
from raptor_validation.raptor import PIM_PASS_LABELS, parse_pipeline_depth
DEFAULT_PIMCOMP_CONFIG = "arch-a"
PIMCOMP_CONFIG_CHOICES = ("arch-a", "arch-b", "arch-c")
@@ -222,6 +222,8 @@ def main():
help="Skip non-functional simulation.")
ap.add_argument("--no-fast", action="store_true",
help="Disable fast Pimsim throughput convergence for authoritative experiments.")
ap.add_argument("--throughput-pipeline", type=parse_pipeline_depth, default=THROUGHPUT_PIPELINE,
help="Throughput pipeline depth, or auto (default: 4).")
ap.add_argument("--threshold", type=float, default=1e-3,
help="Absolute tolerance for per-element output comparison.")
ap.add_argument("--relative-threshold", type=float, default=1e-5,
@@ -345,7 +347,7 @@ def main():
"raptor_extra_args": raptor_extra_args,
"pimsim_nn_build_dir": pimsim_nn_build_dir,
"pimsim_config_path": selected_pimsim_config,
"throughput_pipeline": THROUGHPUT_PIPELINE,
"throughput_pipeline": a.throughput_pipeline,
"throughput_batch_size": THROUGHPUT_BATCH_SIZE,
"throughput_pimsim_config_path": selected_throughput_pimsim_config,
"pimsim_fast": not a.no_fast,