#!/usr/bin/env python3 """Run the synchronization and Spatial-planning ablation matrix on Pimcomp models.""" from __future__ import annotations import argparse import csv import math import os import shlex import signal import subprocess import sys from dataclasses import dataclass from pathlib import Path REPO = Path(__file__).resolve().parents[4] RUNNER = REPO / "validation/tools/pim/pimcomp/compare/run_pimcomp_models.py" DEFAULT_OUT_DIR = REPO / "validation/networks/pimcomp_models" sys.path.insert(0, str(REPO / "validation")) from raptor_validation.pimcomp_models import ( ABLATION_DEFAULT_MODELS, add_models_argument, ) from raptor_validation.artifacts import remove_lock_files from raptor_validation.raptor import parse_pipeline_depth, pipeline_sort_key @dataclass(frozen=True) class Variant: name: str raptor_args: tuple[str, ...] VARIANTS = ( Variant("baseline", ()), Variant("no-sync", ("--pim-disable-synchronization",)), Variant("no-plan", ("--pim-disable-spatial-planning",)), ) VARIANT_BY_NAME = {variant.name: variant for variant in VARIANTS} COMPARISON_RESULTS_FILENAME = "results_comparison.csv" ABLATION_RESULTS_FILENAME = "results_ablation.csv" DEFAULT_RESULTS_PATH = Path(__file__).with_name(ABLATION_RESULTS_FILENAME) CASE_FIELDS = ("arch", "model", "mode", "raptor_pipeline") PERCENTAGE_FIELDS = ( ("raptor_latency_ms", "latency"), ("raptor_throughput_samples_s", "throughput"), ("raptor_power_mw", "power"), ("raptor_energy_pj", "energy"), ) RESULT_FIELDS = ( "comparison", *CASE_FIELDS, "baseline_variant", "baseline_functional_validation", "functional_validation", *(f"{target}_gain_percent" for _, target in PERCENTAGE_FIELDS), ) DEFAULT_CASE_ARGUMENTS = ( ("latency", ("--mode", "latency")), ("throughput/pipeline4", ("--mode", "throughput", "--pipeline", "4")), ) def parse_args(argv: list[str] | None = None) -> tuple[argparse.Namespace, list[str]]: parser = argparse.ArgumentParser( description="Measure planning and synchronization gains on the Pimcomp model suite.", allow_abbrev=False, ) parser.add_argument( "--out-dir", type=Path, help="Custom experiment root for artifacts and aggregate CSV. Default: model-suite artifacts and results_ablation.csv beside this script.", ) add_models_argument(parser, ABLATION_DEFAULT_MODELS) parser.add_argument( "--variant", choices=tuple(VARIANT_BY_NAME), action="append", dest="variants", help="Run only this variant; baseline is added when needed. Repeat as needed.", ) parser.add_argument("--dry-run", action="store_true", help="Print runner commands without modifying files.") args, forwarded = parser.parse_known_args(argv) if any(option == "--only" or option.startswith("--only=") for option in forwarded): parser.error("--only is not supported; the ablation wrapper always runs Raptor only") return args, forwarded def selected_variants(names: list[str] | None) -> list[Variant]: requested = set(names or VARIANT_BY_NAME) requested.add("baseline") return [variant for variant in VARIANTS if variant.name in requested] def has_option(arguments: list[str], option: str) -> bool: return any(argument == option or argument.startswith(f"{option}=") for argument in arguments) def runner_argument_sets(forwarded: list[str]) -> tuple[tuple[str, list[str]], ...]: if not has_option(forwarded, "--mode") and not has_option(forwarded, "--pipeline"): return tuple( (label, [*forwarded, *case_arguments]) for label, case_arguments in DEFAULT_CASE_ARGUMENTS ) parser = argparse.ArgumentParser(add_help=False, allow_abbrev=False) parser.add_argument("--mode", nargs="+", choices=("latency", "throughput")) parser.add_argument("--pipeline", type=parse_pipeline_depth) args, remaining = parser.parse_known_args(forwarded) requested = (("latency", 1), ("throughput", 2), ("throughput", 4), ("throughput", 8)) if args.pipeline is None else ( ("latency", 1) if args.pipeline == 1 else ("throughput", args.pipeline), ) cases = [(mode, pipeline) for mode, pipeline in requested if (args.mode is None or mode in args.mode) and (args.pipeline is None or pipeline == args.pipeline)] if not cases: parser.error("--pipeline does not belong to the selected --mode") return tuple((f"{mode}/pipeline{pipeline}", [*remaining, "--mode", mode, "--pipeline", str(pipeline)]) for mode, pipeline in cases) def case_key(arguments: list[str]) -> tuple[str, str]: mode = arguments[arguments.index("--mode") + 1] pipeline = arguments[arguments.index("--pipeline") + 1] if "--pipeline" in arguments else "1" return mode, pipeline def variant_output_dir(out_dir: Path, variant: Variant) -> Path: return out_dir / "ablation" / variant.name def runner_command( out_dir: Path, variant: Variant, models: list[str], forwarded: list[str], common_root: Path, dry_run: bool, ) -> list[str]: command = [ sys.executable, str(RUNNER), "--out-dir", str(out_dir), "--raptor-only", "--ablation-variant", variant.name, "--models", *models, ] if not has_option(forwarded, "--common-dir"): command.extend(("--common-dir", str(common_root))) command.extend(forwarded) command.extend(f"--raptor-extra-arg={arg}" for arg in variant.raptor_args) if dry_run: command.append("--dry-run") return command def terminate_process_group(process: subprocess.Popen[bytes]) -> None: if process.poll() is not None: return try: os.killpg(process.pid, signal.SIGTERM) except ProcessLookupError: return try: process.wait(timeout=5) except subprocess.TimeoutExpired: try: os.killpg(process.pid, signal.SIGKILL) except ProcessLookupError: return process.wait() def percentage_difference( value: str | None, reference: str | None, *, lower_is_better: bool = False ) -> str: try: current = float(value) if value is not None else math.nan baseline = float(reference) if reference is not None else math.nan except ValueError: return "NA" if not math.isfinite(current) or not math.isfinite(baseline) or current <= 0 or baseline <= 0: return "NA" ratio = current / baseline change = (1 - ratio if lower_is_better else ratio - 1) * 100 return f"{change:+.2f}%" if math.isfinite(change) else "NA" def aggregate_results( out_dir: Path, variants: list[Variant], selected_cases: frozenset[tuple[str, str]] | None = None, selected_models: frozenset[str] | None = None, output: Path | None = None, ) -> tuple[Path | None, list[str]]: fields: list[str] | None = None rows_by_variant: dict[str, list[dict[str, str]]] = {} failures = [] for variant in variants: results_path = variant_output_dir(out_dir, variant) / COMPARISON_RESULTS_FILENAME if not results_path.is_file(): failures.append(f"{variant.name}: missing {results_path}") continue with results_path.open(newline="", encoding="utf-8") as stream: reader = csv.DictReader(stream) if reader.fieldnames is None: failures.append(f"{variant.name}: empty {results_path}") continue if fields is None: fields = reader.fieldnames elif reader.fieldnames != fields: failures.append(f"{variant.name}: inconsistent columns in {results_path}") continue selected_rows = rows_by_variant.setdefault(variant.name, []) for row in reader: if selected_models is not None and row.get("model") not in selected_models: continue if selected_cases is not None and ( row.get("mode"), row.get("raptor_pipeline") ) not in selected_cases: continue selected_rows.append(row) if fields is None: return None, failures enabled_rows = { tuple(row.get(field, "") for field in CASE_FIELDS): row for row in rows_by_variant.get("baseline", []) } if "baseline" not in rows_by_variant: failures.append("baseline: missing fully enabled results") result_rows = [] for variant in variants: if variant.name == "baseline": continue disabled_keys = { tuple(row.get(field, "") for field in CASE_FIELDS) for row in rows_by_variant.get(variant.name, []) } for key, row in enabled_rows.items(): if variant.name == "no-sync" and row.get("mode") != "throughput": continue if key not in disabled_keys: failures.append(f"{variant.name}: missing baseline case " + "/".join(key)) for row in rows_by_variant.get(variant.name, []): if variant.name == "no-sync" and row.get("mode") != "throughput": continue enabled = enabled_rows.get(tuple(row.get(field, "") for field in CASE_FIELDS)) if enabled is None: failures.append(f"{variant.name}: missing fully enabled case " + "/".join(row.get(field, "") for field in CASE_FIELDS)) result_rows.append( { "comparison": "plan" if variant.name == "no-plan" else "sync", **{field: row.get(field, "") for field in CASE_FIELDS}, "baseline_variant": variant.name, "baseline_functional_validation": row.get("raptor_functional_validation", "NA"), "functional_validation": enabled.get("raptor_functional_validation", "NA") if enabled else "NA", **{ f"{target}_gain_percent": percentage_difference( enabled.get(source) if enabled else None, row.get(source), lower_is_better=target != "throughput", ) for source, target in PERCENTAGE_FIELDS }, } ) result_rows.sort( key=lambda row: ( row["arch"], row["model"], row["mode"], pipeline_sort_key(row["raptor_pipeline"]), row["comparison"], ) ) output = output if output is not None else out_dir / ABLATION_RESULTS_FILENAME output.parent.mkdir(parents=True, exist_ok=True) with output.open("w", newline="", encoding="utf-8") as stream: writer = csv.DictWriter(stream, fieldnames=RESULT_FIELDS, lineterminator="\n") writer.writeheader() writer.writerows(result_rows) return output, failures def main(argv: list[str] | None = None) -> int: args, forwarded = parse_args(argv) if not RUNNER.is_file(): print(f"Missing Pimcomp runner: {RUNNER}", file=sys.stderr) return 1 variants = selected_variants(args.variants) out_dir = args.out_dir.resolve() if args.out_dir is not None else DEFAULT_OUT_DIR case_sets = runner_argument_sets(forwarded) if not any(case_key(arguments)[0] == "throughput" for _, arguments in case_sets): variants = [variant for variant in variants if variant.name != "no-sync"] if args.dry_run: for variant in variants: for _, case_forwarded in case_sets: if variant.name == "no-sync" and case_key(case_forwarded)[0] != "throughput": continue print( shlex.join( runner_command( out_dir, variant, args.models, case_forwarded, out_dir, True, ) ) ) return 0 out_dir.mkdir(parents=True, exist_ok=True) failed = [] current_process = None try: for variant in variants: for case_label, case_forwarded in case_sets: if variant.name == "no-sync" and case_key(case_forwarded)[0] != "throughput": continue command = runner_command( out_dir, variant, args.models, case_forwarded, out_dir, False ) print(f"[{variant.name}/{case_label}] {shlex.join(command)}") current_process = subprocess.Popen( command, cwd=REPO, start_new_session=True, ) returncode = current_process.wait() current_process = None if returncode: failed.append(f"{variant.name}/{case_label}: runner exited with {returncode}") except KeyboardInterrupt: if current_process is not None: terminate_process_group(current_process) remove_lock_files(out_dir) print("Interrupted; terminated the active ablation job.", file=sys.stderr) return 130 remove_lock_files(out_dir) selected_cases = frozenset(case_key(arguments) for _, arguments in case_sets) output, aggregation_failures = aggregate_results( out_dir, variants, selected_cases, frozenset(args.models), output=DEFAULT_RESULTS_PATH if args.out_dir is None else None, ) failed.extend(aggregation_failures) if output is not None: print(f"[{output.name}]") print(output.read_text(encoding="utf-8"), end="", flush=True) if failed: print("Failed: " + "; ".join(failed), file=sys.stderr) if output is not None: print(f"Ablation results: {output}", flush=True) return 1 if failed else 0 if __name__ == "__main__": raise SystemExit(main())