force selected pipeline stages in throughput mode
Validate Operations / validate-operations (push) Has been cancelled

increase throughput scheduler budget
add experimental throughput auto mode
This commit is contained in:
NiccoloN
2026-11-02 12:08:33 +01:00
parent 4168b588fa
commit 4a1fe710b2
35 changed files with 2047 additions and 398 deletions
+2
View File
@@ -31,6 +31,8 @@ Other [suite-runner options](../pimcomp/compare/README.md), including `--models`
`--archs`, and `--no-fast`, are forwarded. Supplying `--mode` or `--pipeline`
replaces the default latency/pipeline-4 case set. Mode-only selection runs all
suite pipelines for that mode. Pipeline-only selection uses its matching mode.
`--pipeline auto` requests automatic throughput depth selection; positive numeric
depths retain their exact-stage meaning.
`--only` is rejected because
the wrapper always runs Raptor only. Ctrl+C stops the active variant's workers.
@@ -25,6 +25,7 @@ from raptor_validation.pimcomp_models import (
add_models_argument,
)
from raptor_validation.artifacts import remove_lock_files
from raptor_validation.raptor import parse_pipeline_depth, pipeline_sort_key
@dataclass(frozen=True)
@@ -102,10 +103,12 @@ def runner_argument_sets(forwarded: list[str]) -> tuple[tuple[str, list[str]], .
)
parser = argparse.ArgumentParser(add_help=False, allow_abbrev=False)
parser.add_argument("--mode", nargs="+", choices=("latency", "throughput"))
parser.add_argument("--pipeline", type=int, choices=(1, 2, 4, 8))
parser.add_argument("--pipeline", type=parse_pipeline_depth)
args, remaining = parser.parse_known_args(forwarded)
cases = [(mode, pipeline) for mode, pipeline in
(("latency", 1), ("throughput", 2), ("throughput", 4), ("throughput", 8))
requested = (("latency", 1), ("throughput", 2), ("throughput", 4), ("throughput", 8)) if args.pipeline is None else (
("latency", 1) if args.pipeline == 1 else ("throughput", args.pipeline),
)
cases = [(mode, pipeline) for mode, pipeline in requested
if (args.mode is None or mode in args.mode)
and (args.pipeline is None or pipeline == args.pipeline)]
if not cases:
@@ -268,7 +271,7 @@ def aggregate_results(
row["arch"],
row["model"],
row["mode"],
int(row["raptor_pipeline"]),
pipeline_sort_key(row["raptor_pipeline"]),
row["comparison"],
)
)
@@ -12,12 +12,15 @@ built release Raptor, Pimcomp, and `pimsim-nn`.
Defaults: all five models, Arch-A/B, latency pipeline 1, throughput pipelines
2/4/8, four workers, and a functional throughput batch of 64.
Use `--help` for the complete option reference.
Use `--help` for the complete option reference. Numeric depths greater than one
require exactly that many active compute stages. `--pipeline auto` requests
throughput depth selection. Reports include `raptor_selected_pipeline`; the
requested depth remains in the existing pipeline field.
| Option | Purpose |
|---|---|
| `--models MODEL [...]`, `--archs ARCH [...]` | Select models and hardware profiles. |
| `--mode {latency,throughput} [...]`, `--pipeline {1,2,4,8}` | Select experiments. |
| `--mode {latency,throughput} [...]`, `--pipeline N\|auto` | Select experiments. |
| `--out-dir PATH` | Suite root; default: `validation/networks/pimcomp_models`. |
| `--common-dir PATH` | Override the shared reference-artifact root. |
| `--only {raptor,pimcomp}` | Rerun one compiler and reuse the other's existing report. |
@@ -44,7 +44,7 @@ from raptor_validation.onnx_utils import (
write_input_batch_csv,
write_inputs_to_memory_bin,
)
from raptor_validation.raptor import compile_with_raptor
from raptor_validation.raptor import compile_with_raptor, read_selected_pipeline_depth
from raptor_validation.output_comparison import VALIDATION_POLICY, compare_output_arrays
from raptor_validation.pimsim_nn import (
export_raptor_pimsim_artifact,
@@ -2160,6 +2160,7 @@ def main():
"raptor_reuse_context": experiment_context(args, model_path, functional_model_path, hardware, "raptor"),
"pimcomp_config": str(args.pimcomp_config),
"raptor_extra_args": args.raptor_extra_arg,
"raptor_selected_pipeline": read_selected_pipeline_depth(raptor_pim_dir),
"reused_raptor_report": optional_path(args.reuse_raptor_report.resolve()) if reuse_raptor else None,
"reused_pimcomp_report": optional_path(reused_pimcomp_report_path) if reuse_pimcomp else None,
"failures": failures,
@@ -30,6 +30,7 @@ from raptor_validation.pimcomp_models import (
from raptor_validation.artifacts import artifacts_dir, remove_lock_files
from raptor_validation.pimsim_nn import parse_pimsim_nn_metrics
from raptor_validation.output_comparison import VALIDATION_POLICY
from raptor_validation.raptor import parse_pipeline_depth, pipeline_sort_key
from raptor_validation.validate_one import STAGE_COLORS
PIMCOMP_SOURCE = REPO / "third_party/PIMCOMP-NN"
@@ -56,7 +57,7 @@ class ComparisonSpec:
common_dir: Path
config: Path
mode: str
pipeline: int
pipeline: int | str
pimcomp_pipeline: str
shared_pimcomp_dir: Path
anchor: bool
@@ -71,7 +72,7 @@ def result_dir(
name: str,
arch: str,
mode: str,
pipeline: int,
pipeline: int | str,
ablation_variant: str | None = None,
) -> Path:
base = artifacts_dir(model_dir(root, name))
@@ -243,7 +244,7 @@ def write_results_csv(
row["arch"],
model_order[row["model"]],
mode_order[row["mode"]],
int(row["raptor_pipeline"]),
pipeline_sort_key(row["raptor_pipeline"]),
)
)
output.parent.mkdir(parents=True, exist_ok=True)
@@ -378,7 +379,7 @@ def comparison_command(
common_dir: Path,
config: Path,
mode: str,
pipeline: int,
pipeline: int | str,
pimcomp_pipeline: str,
pimsim_time_ms: int,
batch_size: int,
@@ -610,9 +611,8 @@ def main() -> int:
)
parser.add_argument(
"--pipeline",
type=int,
choices=(1, 2, 4, 8),
help="Run only this Raptor pipeline within the selected mode.",
type=parse_pipeline_depth,
help="Run this positive Raptor pipeline depth, or auto, within the selected mode.",
)
parser.add_argument(
"--pimsim-time-ms",
@@ -684,11 +684,15 @@ def main() -> int:
parser.error("--batch-size must be positive")
if args.timeout_seconds < 0:
parser.error("--timeout-seconds must be non-negative")
comparisons_by_arch: dict[str, tuple[tuple[str, int, str], ...]] = {}
comparisons_by_arch: dict[str, tuple[tuple[str, int | str, str], ...]] = {}
configs_by_arch: dict[str, dict[str, Path]] = {}
for arch in args.archs:
requested = COMPARISONS if args.pipeline is None else (
("latency", 1, "element") if args.pipeline == 1
else ("throughput", args.pipeline, "batch"),
)
comparisons = tuple(
comparison for comparison in COMPARISONS
comparison for comparison in requested
if comparison[0] in args.mode
and (args.pipeline is None or comparison[1] == args.pipeline)
)