force selected pipeline stages in throughput mode
Validate Operations / validate-operations (push) Has been cancelled
Validate Operations / validate-operations (push) Has been cancelled
increase throughput scheduler budget add experimental throughput auto mode
This commit is contained in:
@@ -31,6 +31,8 @@ Other [suite-runner options](../pimcomp/compare/README.md), including `--models`
|
||||
`--archs`, and `--no-fast`, are forwarded. Supplying `--mode` or `--pipeline`
|
||||
replaces the default latency/pipeline-4 case set. Mode-only selection runs all
|
||||
suite pipelines for that mode. Pipeline-only selection uses its matching mode.
|
||||
`--pipeline auto` requests automatic throughput depth selection; positive numeric
|
||||
depths retain their exact-stage meaning.
|
||||
`--only` is rejected because
|
||||
the wrapper always runs Raptor only. Ctrl+C stops the active variant's workers.
|
||||
|
||||
|
||||
@@ -25,6 +25,7 @@ from raptor_validation.pimcomp_models import (
|
||||
add_models_argument,
|
||||
)
|
||||
from raptor_validation.artifacts import remove_lock_files
|
||||
from raptor_validation.raptor import parse_pipeline_depth, pipeline_sort_key
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -102,10 +103,12 @@ def runner_argument_sets(forwarded: list[str]) -> tuple[tuple[str, list[str]], .
|
||||
)
|
||||
parser = argparse.ArgumentParser(add_help=False, allow_abbrev=False)
|
||||
parser.add_argument("--mode", nargs="+", choices=("latency", "throughput"))
|
||||
parser.add_argument("--pipeline", type=int, choices=(1, 2, 4, 8))
|
||||
parser.add_argument("--pipeline", type=parse_pipeline_depth)
|
||||
args, remaining = parser.parse_known_args(forwarded)
|
||||
cases = [(mode, pipeline) for mode, pipeline in
|
||||
(("latency", 1), ("throughput", 2), ("throughput", 4), ("throughput", 8))
|
||||
requested = (("latency", 1), ("throughput", 2), ("throughput", 4), ("throughput", 8)) if args.pipeline is None else (
|
||||
("latency", 1) if args.pipeline == 1 else ("throughput", args.pipeline),
|
||||
)
|
||||
cases = [(mode, pipeline) for mode, pipeline in requested
|
||||
if (args.mode is None or mode in args.mode)
|
||||
and (args.pipeline is None or pipeline == args.pipeline)]
|
||||
if not cases:
|
||||
@@ -268,7 +271,7 @@ def aggregate_results(
|
||||
row["arch"],
|
||||
row["model"],
|
||||
row["mode"],
|
||||
int(row["raptor_pipeline"]),
|
||||
pipeline_sort_key(row["raptor_pipeline"]),
|
||||
row["comparison"],
|
||||
)
|
||||
)
|
||||
|
||||
@@ -12,12 +12,15 @@ built release Raptor, Pimcomp, and `pimsim-nn`.
|
||||
|
||||
Defaults: all five models, Arch-A/B, latency pipeline 1, throughput pipelines
|
||||
2/4/8, four workers, and a functional throughput batch of 64.
|
||||
Use `--help` for the complete option reference.
|
||||
Use `--help` for the complete option reference. Numeric depths greater than one
|
||||
require exactly that many active compute stages. `--pipeline auto` requests
|
||||
throughput depth selection. Reports include `raptor_selected_pipeline`; the
|
||||
requested depth remains in the existing pipeline field.
|
||||
|
||||
| Option | Purpose |
|
||||
|---|---|
|
||||
| `--models MODEL [...]`, `--archs ARCH [...]` | Select models and hardware profiles. |
|
||||
| `--mode {latency,throughput} [...]`, `--pipeline {1,2,4,8}` | Select experiments. |
|
||||
| `--mode {latency,throughput} [...]`, `--pipeline N\|auto` | Select experiments. |
|
||||
| `--out-dir PATH` | Suite root; default: `validation/networks/pimcomp_models`. |
|
||||
| `--common-dir PATH` | Override the shared reference-artifact root. |
|
||||
| `--only {raptor,pimcomp}` | Rerun one compiler and reuse the other's existing report. |
|
||||
|
||||
@@ -44,7 +44,7 @@ from raptor_validation.onnx_utils import (
|
||||
write_input_batch_csv,
|
||||
write_inputs_to_memory_bin,
|
||||
)
|
||||
from raptor_validation.raptor import compile_with_raptor
|
||||
from raptor_validation.raptor import compile_with_raptor, read_selected_pipeline_depth
|
||||
from raptor_validation.output_comparison import VALIDATION_POLICY, compare_output_arrays
|
||||
from raptor_validation.pimsim_nn import (
|
||||
export_raptor_pimsim_artifact,
|
||||
@@ -2160,6 +2160,7 @@ def main():
|
||||
"raptor_reuse_context": experiment_context(args, model_path, functional_model_path, hardware, "raptor"),
|
||||
"pimcomp_config": str(args.pimcomp_config),
|
||||
"raptor_extra_args": args.raptor_extra_arg,
|
||||
"raptor_selected_pipeline": read_selected_pipeline_depth(raptor_pim_dir),
|
||||
"reused_raptor_report": optional_path(args.reuse_raptor_report.resolve()) if reuse_raptor else None,
|
||||
"reused_pimcomp_report": optional_path(reused_pimcomp_report_path) if reuse_pimcomp else None,
|
||||
"failures": failures,
|
||||
|
||||
@@ -30,6 +30,7 @@ from raptor_validation.pimcomp_models import (
|
||||
from raptor_validation.artifacts import artifacts_dir, remove_lock_files
|
||||
from raptor_validation.pimsim_nn import parse_pimsim_nn_metrics
|
||||
from raptor_validation.output_comparison import VALIDATION_POLICY
|
||||
from raptor_validation.raptor import parse_pipeline_depth, pipeline_sort_key
|
||||
from raptor_validation.validate_one import STAGE_COLORS
|
||||
|
||||
PIMCOMP_SOURCE = REPO / "third_party/PIMCOMP-NN"
|
||||
@@ -56,7 +57,7 @@ class ComparisonSpec:
|
||||
common_dir: Path
|
||||
config: Path
|
||||
mode: str
|
||||
pipeline: int
|
||||
pipeline: int | str
|
||||
pimcomp_pipeline: str
|
||||
shared_pimcomp_dir: Path
|
||||
anchor: bool
|
||||
@@ -71,7 +72,7 @@ def result_dir(
|
||||
name: str,
|
||||
arch: str,
|
||||
mode: str,
|
||||
pipeline: int,
|
||||
pipeline: int | str,
|
||||
ablation_variant: str | None = None,
|
||||
) -> Path:
|
||||
base = artifacts_dir(model_dir(root, name))
|
||||
@@ -243,7 +244,7 @@ def write_results_csv(
|
||||
row["arch"],
|
||||
model_order[row["model"]],
|
||||
mode_order[row["mode"]],
|
||||
int(row["raptor_pipeline"]),
|
||||
pipeline_sort_key(row["raptor_pipeline"]),
|
||||
)
|
||||
)
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
@@ -378,7 +379,7 @@ def comparison_command(
|
||||
common_dir: Path,
|
||||
config: Path,
|
||||
mode: str,
|
||||
pipeline: int,
|
||||
pipeline: int | str,
|
||||
pimcomp_pipeline: str,
|
||||
pimsim_time_ms: int,
|
||||
batch_size: int,
|
||||
@@ -610,9 +611,8 @@ def main() -> int:
|
||||
)
|
||||
parser.add_argument(
|
||||
"--pipeline",
|
||||
type=int,
|
||||
choices=(1, 2, 4, 8),
|
||||
help="Run only this Raptor pipeline within the selected mode.",
|
||||
type=parse_pipeline_depth,
|
||||
help="Run this positive Raptor pipeline depth, or auto, within the selected mode.",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--pimsim-time-ms",
|
||||
@@ -684,11 +684,15 @@ def main() -> int:
|
||||
parser.error("--batch-size must be positive")
|
||||
if args.timeout_seconds < 0:
|
||||
parser.error("--timeout-seconds must be non-negative")
|
||||
comparisons_by_arch: dict[str, tuple[tuple[str, int, str], ...]] = {}
|
||||
comparisons_by_arch: dict[str, tuple[tuple[str, int | str, str], ...]] = {}
|
||||
configs_by_arch: dict[str, dict[str, Path]] = {}
|
||||
for arch in args.archs:
|
||||
requested = COMPARISONS if args.pipeline is None else (
|
||||
("latency", 1, "element") if args.pipeline == 1
|
||||
else ("throughput", args.pipeline, "batch"),
|
||||
)
|
||||
comparisons = tuple(
|
||||
comparison for comparison in COMPARISONS
|
||||
comparison for comparison in requested
|
||||
if comparison[0] in args.mode
|
||||
and (args.pipeline is None or comparison[1] == args.pipeline)
|
||||
)
|
||||
|
||||
Reference in New Issue
Block a user