force selected pipeline stages in throughput mode
Validate Operations / validate-operations (push) Has been cancelled
Validate Operations / validate-operations (push) Has been cancelled
increase throughput scheduler budget add experimental throughput auto mode
This commit is contained in:
@@ -112,6 +112,12 @@ The regular validator uses these paths beside each model:
|
||||
| `artifacts/raptor/throughput/pim/` | Pipeline-4 throughput artifacts. |
|
||||
| `artifacts/simulation/{latency,throughput}/` | Functional simulator outputs. |
|
||||
|
||||
`--throughput-pipeline=N` selects an exact throughput depth; `auto` requests
|
||||
automatic depth selection. The default remains 4. Compiler logs report the
|
||||
selected depth when a selection report is available. Small operation graphs can
|
||||
reject exact depth 4. Use `--throughput-pipeline=auto` to test their feasible
|
||||
throughput depth without weakening explicit-depth constraints.
|
||||
|
||||
The [comparison runner](tools/pim/pimcomp/compare/README.md) has separate
|
||||
architecture/pipeline directories and shared `artifacts/common/` references.
|
||||
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import argparse
|
||||
import json
|
||||
import re
|
||||
import shlex
|
||||
import shutil
|
||||
@@ -20,6 +22,29 @@ PIM_PASS_LABEL_BY_SUFFIX = dict(PIM_PASS_LABELS)
|
||||
TIMING_LINE_RE = re.compile(r"^\s*([0-9]+\.[0-9]+)\s+\(\s*[0-9.]+%\)\s+(.+?)\s*$")
|
||||
|
||||
|
||||
def parse_pipeline_depth(value):
|
||||
if value == "auto":
|
||||
return value
|
||||
try:
|
||||
depth = int(value)
|
||||
except ValueError:
|
||||
raise argparse.ArgumentTypeError("pipeline must be a positive integer or auto") from None
|
||||
if depth < 1:
|
||||
raise argparse.ArgumentTypeError("pipeline must be a positive integer or auto")
|
||||
return depth
|
||||
|
||||
|
||||
def pipeline_sort_key(value):
|
||||
return (str(value) == "auto", 0 if str(value) == "auto" else int(value))
|
||||
|
||||
|
||||
def read_selected_pipeline_depth(pim_dir):
|
||||
if pim_dir is None:
|
||||
return None
|
||||
report = Path(pim_dir).parent / "reports/pipeline_selection.json"
|
||||
return json.loads(report.read_text()).get("selected_depth") if report.exists() else None
|
||||
|
||||
|
||||
def _parse_pim_pass_timings(output_text):
|
||||
pass_timings = {}
|
||||
for line in output_text.splitlines():
|
||||
@@ -79,6 +104,13 @@ def compile_with_raptor(network_path, raptor_onnx_path: Path, output_base: Path,
|
||||
capture_output=True,
|
||||
timeout_sec=timeout_sec,
|
||||
)
|
||||
selected_depth = read_selected_pipeline_depth(pim_dir)
|
||||
if selected_depth is not None:
|
||||
message = f"Selected pipeline depth: {selected_depth}"
|
||||
if reporter is not None:
|
||||
reporter.log(message)
|
||||
else:
|
||||
print(message)
|
||||
if reporter is None:
|
||||
print(Fore.GREEN + "Raptor execution successful" + Style.RESET_ALL)
|
||||
return _parse_pim_pass_timings(output_text)
|
||||
|
||||
@@ -20,9 +20,80 @@ sys.path.insert(0, str(TOOLS / "pimcomp/compare"))
|
||||
import run_ablation as ablation
|
||||
import run_pimcomp_models as suite
|
||||
import compare_raptor_pimcomp_model as comparator
|
||||
from raptor_validation.raptor import parse_pipeline_depth, pipeline_sort_key, read_selected_pipeline_depth
|
||||
from raptor_validation.validate_one import validate_outputs
|
||||
|
||||
|
||||
def test_default_experiment_depths_exclude_auto():
|
||||
expected = [("latency", 1), ("throughput", 2), ("throughput", 4), ("throughput", 8)]
|
||||
assert [(mode, depth) for mode, depth, _ in suite.COMPARISONS] == expected
|
||||
cases = ablation.runner_argument_sets(["--mode", "latency", "throughput"])
|
||||
assert [ablation.case_key(arguments) for _, arguments in cases] == [
|
||||
(mode, str(depth)) for mode, depth in expected]
|
||||
assert all("auto" not in arguments for _, arguments in ablation.runner_argument_sets([]))
|
||||
|
||||
|
||||
def test_operation_validation_forwards_requested_depth(tmp_path, monkeypatch):
|
||||
import importlib.util
|
||||
|
||||
spec = importlib.util.spec_from_file_location("operation_validation", TOOLS.parents[1] / "validate.py")
|
||||
operation_validation = importlib.util.module_from_spec(spec)
|
||||
spec.loader.exec_module(operation_validation)
|
||||
model = tmp_path / "model.onnx"
|
||||
model.touch()
|
||||
monkeypatch.setattr(operation_validation, "discover_onnx_files", lambda _: [model])
|
||||
monkeypatch.setattr(sys, "argv", ["validate.py", "--operations-dir", str(tmp_path),
|
||||
"--raptor-path", str(tmp_path / "compiler"), "--onnx-include-dir", str(tmp_path),
|
||||
"--skip-non-functional-simulation", "--jobs", "1", "--throughput-pipeline", "8"])
|
||||
|
||||
class JobReached(Exception):
|
||||
pass
|
||||
|
||||
def check_job(job):
|
||||
assert job[1]["validation_kwargs"]["throughput_pipeline"] == 8
|
||||
raise JobReached
|
||||
|
||||
monkeypatch.setattr(operation_validation, "run_validation_job", check_job)
|
||||
with pytest.raises(JobReached):
|
||||
operation_validation.main()
|
||||
|
||||
|
||||
@pytest.mark.parametrize("depth", ["auto", "3"])
|
||||
def test_comparison_and_ablation_accept_depth_domain(tmp_path, monkeypatch, depth):
|
||||
commands = []
|
||||
monkeypatch.setattr(suite, "run", lambda command, **kwargs: commands.append(command) or 0)
|
||||
monkeypatch.setattr(sys, "argv", ["run_pimcomp_models.py", "--out-dir", str(tmp_path),
|
||||
"--models", "vgg8", "--archs", "arch-b", "--mode", "throughput", "--pipeline", depth,
|
||||
"--raptor-only", "--dry-run"])
|
||||
assert suite.main() == 0
|
||||
assert len(commands) == 2
|
||||
assert f"--raptor-extra-arg=--pipeline={depth}" in commands[-1]
|
||||
assert f"pipeline{depth}" in commands[-1][commands[-1].index("--out-dir") + 1]
|
||||
cases = ablation.runner_argument_sets(["--mode", "throughput", "--pipeline", depth])
|
||||
assert len(cases) == 1 and cases[0][0] == f"throughput/pipeline{depth}"
|
||||
assert cases[0][1][-1] == depth
|
||||
|
||||
|
||||
def test_pipeline_depth_parser():
|
||||
for invalid in ["0", "-1", "unknown", "2.5"]:
|
||||
with pytest.raises(argparse.ArgumentTypeError):
|
||||
parse_pipeline_depth(invalid)
|
||||
assert parse_pipeline_depth("1") == 1
|
||||
assert parse_pipeline_depth("auto") == "auto"
|
||||
assert sorted(["auto", "8", "1", "3"], key=pipeline_sort_key) == ["1", "3", "8", "auto"]
|
||||
|
||||
|
||||
def test_selected_pipeline_report(tmp_path):
|
||||
assert read_selected_pipeline_depth(None) is None
|
||||
assert read_selected_pipeline_depth(tmp_path / "pim") is None
|
||||
(tmp_path / "reports").mkdir()
|
||||
report = tmp_path / "reports/pipeline_selection.json"
|
||||
report.write_text(json.dumps({"requested_depth": "auto", "selected_depth": 3}))
|
||||
assert read_selected_pipeline_depth(tmp_path / "pim") == 3
|
||||
report.write_text(json.dumps({"requested_depth": 8}))
|
||||
assert read_selected_pipeline_depth(tmp_path / "pim") is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("mode", ["latency", "throughput"])
|
||||
def test_comparison_report_energy_is_per_inference(tmp_path, mode):
|
||||
report = tmp_path / "comparison_report.md"
|
||||
|
||||
@@ -31,6 +31,8 @@ Other [suite-runner options](../pimcomp/compare/README.md), including `--models`
|
||||
`--archs`, and `--no-fast`, are forwarded. Supplying `--mode` or `--pipeline`
|
||||
replaces the default latency/pipeline-4 case set. Mode-only selection runs all
|
||||
suite pipelines for that mode. Pipeline-only selection uses its matching mode.
|
||||
`--pipeline auto` requests automatic throughput depth selection; positive numeric
|
||||
depths retain their exact-stage meaning.
|
||||
`--only` is rejected because
|
||||
the wrapper always runs Raptor only. Ctrl+C stops the active variant's workers.
|
||||
|
||||
|
||||
@@ -25,6 +25,7 @@ from raptor_validation.pimcomp_models import (
|
||||
add_models_argument,
|
||||
)
|
||||
from raptor_validation.artifacts import remove_lock_files
|
||||
from raptor_validation.raptor import parse_pipeline_depth, pipeline_sort_key
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -102,10 +103,12 @@ def runner_argument_sets(forwarded: list[str]) -> tuple[tuple[str, list[str]], .
|
||||
)
|
||||
parser = argparse.ArgumentParser(add_help=False, allow_abbrev=False)
|
||||
parser.add_argument("--mode", nargs="+", choices=("latency", "throughput"))
|
||||
parser.add_argument("--pipeline", type=int, choices=(1, 2, 4, 8))
|
||||
parser.add_argument("--pipeline", type=parse_pipeline_depth)
|
||||
args, remaining = parser.parse_known_args(forwarded)
|
||||
cases = [(mode, pipeline) for mode, pipeline in
|
||||
(("latency", 1), ("throughput", 2), ("throughput", 4), ("throughput", 8))
|
||||
requested = (("latency", 1), ("throughput", 2), ("throughput", 4), ("throughput", 8)) if args.pipeline is None else (
|
||||
("latency", 1) if args.pipeline == 1 else ("throughput", args.pipeline),
|
||||
)
|
||||
cases = [(mode, pipeline) for mode, pipeline in requested
|
||||
if (args.mode is None or mode in args.mode)
|
||||
and (args.pipeline is None or pipeline == args.pipeline)]
|
||||
if not cases:
|
||||
@@ -268,7 +271,7 @@ def aggregate_results(
|
||||
row["arch"],
|
||||
row["model"],
|
||||
row["mode"],
|
||||
int(row["raptor_pipeline"]),
|
||||
pipeline_sort_key(row["raptor_pipeline"]),
|
||||
row["comparison"],
|
||||
)
|
||||
)
|
||||
|
||||
@@ -12,12 +12,15 @@ built release Raptor, Pimcomp, and `pimsim-nn`.
|
||||
|
||||
Defaults: all five models, Arch-A/B, latency pipeline 1, throughput pipelines
|
||||
2/4/8, four workers, and a functional throughput batch of 64.
|
||||
Use `--help` for the complete option reference.
|
||||
Use `--help` for the complete option reference. Numeric depths greater than one
|
||||
require exactly that many active compute stages. `--pipeline auto` requests
|
||||
throughput depth selection. Reports include `raptor_selected_pipeline`; the
|
||||
requested depth remains in the existing pipeline field.
|
||||
|
||||
| Option | Purpose |
|
||||
|---|---|
|
||||
| `--models MODEL [...]`, `--archs ARCH [...]` | Select models and hardware profiles. |
|
||||
| `--mode {latency,throughput} [...]`, `--pipeline {1,2,4,8}` | Select experiments. |
|
||||
| `--mode {latency,throughput} [...]`, `--pipeline N\|auto` | Select experiments. |
|
||||
| `--out-dir PATH` | Suite root; default: `validation/networks/pimcomp_models`. |
|
||||
| `--common-dir PATH` | Override the shared reference-artifact root. |
|
||||
| `--only {raptor,pimcomp}` | Rerun one compiler and reuse the other's existing report. |
|
||||
|
||||
@@ -44,7 +44,7 @@ from raptor_validation.onnx_utils import (
|
||||
write_input_batch_csv,
|
||||
write_inputs_to_memory_bin,
|
||||
)
|
||||
from raptor_validation.raptor import compile_with_raptor
|
||||
from raptor_validation.raptor import compile_with_raptor, read_selected_pipeline_depth
|
||||
from raptor_validation.output_comparison import VALIDATION_POLICY, compare_output_arrays
|
||||
from raptor_validation.pimsim_nn import (
|
||||
export_raptor_pimsim_artifact,
|
||||
@@ -2160,6 +2160,7 @@ def main():
|
||||
"raptor_reuse_context": experiment_context(args, model_path, functional_model_path, hardware, "raptor"),
|
||||
"pimcomp_config": str(args.pimcomp_config),
|
||||
"raptor_extra_args": args.raptor_extra_arg,
|
||||
"raptor_selected_pipeline": read_selected_pipeline_depth(raptor_pim_dir),
|
||||
"reused_raptor_report": optional_path(args.reuse_raptor_report.resolve()) if reuse_raptor else None,
|
||||
"reused_pimcomp_report": optional_path(reused_pimcomp_report_path) if reuse_pimcomp else None,
|
||||
"failures": failures,
|
||||
|
||||
@@ -30,6 +30,7 @@ from raptor_validation.pimcomp_models import (
|
||||
from raptor_validation.artifacts import artifacts_dir, remove_lock_files
|
||||
from raptor_validation.pimsim_nn import parse_pimsim_nn_metrics
|
||||
from raptor_validation.output_comparison import VALIDATION_POLICY
|
||||
from raptor_validation.raptor import parse_pipeline_depth, pipeline_sort_key
|
||||
from raptor_validation.validate_one import STAGE_COLORS
|
||||
|
||||
PIMCOMP_SOURCE = REPO / "third_party/PIMCOMP-NN"
|
||||
@@ -56,7 +57,7 @@ class ComparisonSpec:
|
||||
common_dir: Path
|
||||
config: Path
|
||||
mode: str
|
||||
pipeline: int
|
||||
pipeline: int | str
|
||||
pimcomp_pipeline: str
|
||||
shared_pimcomp_dir: Path
|
||||
anchor: bool
|
||||
@@ -71,7 +72,7 @@ def result_dir(
|
||||
name: str,
|
||||
arch: str,
|
||||
mode: str,
|
||||
pipeline: int,
|
||||
pipeline: int | str,
|
||||
ablation_variant: str | None = None,
|
||||
) -> Path:
|
||||
base = artifacts_dir(model_dir(root, name))
|
||||
@@ -243,7 +244,7 @@ def write_results_csv(
|
||||
row["arch"],
|
||||
model_order[row["model"]],
|
||||
mode_order[row["mode"]],
|
||||
int(row["raptor_pipeline"]),
|
||||
pipeline_sort_key(row["raptor_pipeline"]),
|
||||
)
|
||||
)
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
@@ -378,7 +379,7 @@ def comparison_command(
|
||||
common_dir: Path,
|
||||
config: Path,
|
||||
mode: str,
|
||||
pipeline: int,
|
||||
pipeline: int | str,
|
||||
pimcomp_pipeline: str,
|
||||
pimsim_time_ms: int,
|
||||
batch_size: int,
|
||||
@@ -610,9 +611,8 @@ def main() -> int:
|
||||
)
|
||||
parser.add_argument(
|
||||
"--pipeline",
|
||||
type=int,
|
||||
choices=(1, 2, 4, 8),
|
||||
help="Run only this Raptor pipeline within the selected mode.",
|
||||
type=parse_pipeline_depth,
|
||||
help="Run this positive Raptor pipeline depth, or auto, within the selected mode.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--pimsim-time-ms",
|
||||
@@ -684,11 +684,15 @@ def main() -> int:
|
||||
parser.error("--batch-size must be positive")
|
||||
if args.timeout_seconds < 0:
|
||||
parser.error("--timeout-seconds must be non-negative")
|
||||
comparisons_by_arch: dict[str, tuple[tuple[str, int, str], ...]] = {}
|
||||
comparisons_by_arch: dict[str, tuple[tuple[str, int | str, str], ...]] = {}
|
||||
configs_by_arch: dict[str, dict[str, Path]] = {}
|
||||
for arch in args.archs:
|
||||
requested = COMPARISONS if args.pipeline is None else (
|
||||
("latency", 1, "element") if args.pipeline == 1
|
||||
else ("throughput", args.pipeline, "batch"),
|
||||
)
|
||||
comparisons = tuple(
|
||||
comparison for comparison in COMPARISONS
|
||||
comparison for comparison in requested
|
||||
if comparison[0] in args.mode
|
||||
and (args.pipeline is None or comparison[1] == args.pipeline)
|
||||
)
|
||||
|
||||
@@ -30,7 +30,7 @@ from raptor_validation.validate_one import (
|
||||
pimcomp_compatibility_errors,
|
||||
validate_network,
|
||||
)
|
||||
from raptor_validation.raptor import PIM_PASS_LABELS
|
||||
from raptor_validation.raptor import PIM_PASS_LABELS, parse_pipeline_depth
|
||||
|
||||
DEFAULT_PIMCOMP_CONFIG = "arch-a"
|
||||
PIMCOMP_CONFIG_CHOICES = ("arch-a", "arch-b", "arch-c")
|
||||
@@ -222,6 +222,8 @@ def main():
|
||||
help="Skip non-functional simulation.")
|
||||
ap.add_argument("--no-fast", action="store_true",
|
||||
help="Disable fast Pimsim throughput convergence for authoritative experiments.")
|
||||
ap.add_argument("--throughput-pipeline", type=parse_pipeline_depth, default=THROUGHPUT_PIPELINE,
|
||||
help="Throughput pipeline depth, or auto (default: 4).")
|
||||
ap.add_argument("--threshold", type=float, default=1e-3,
|
||||
help="Absolute tolerance for per-element output comparison.")
|
||||
ap.add_argument("--relative-threshold", type=float, default=1e-5,
|
||||
@@ -345,7 +347,7 @@ def main():
|
||||
"raptor_extra_args": raptor_extra_args,
|
||||
"pimsim_nn_build_dir": pimsim_nn_build_dir,
|
||||
"pimsim_config_path": selected_pimsim_config,
|
||||
"throughput_pipeline": THROUGHPUT_PIPELINE,
|
||||
"throughput_pipeline": a.throughput_pipeline,
|
||||
"throughput_batch_size": THROUGHPUT_BATCH_SIZE,
|
||||
"throughput_pimsim_config_path": selected_throughput_pimsim_config,
|
||||
"pimsim_fast": not a.no_fast,
|
||||
|
||||
Reference in New Issue
Block a user