more complete pimcomp comparison scripts

update pimsim-nn submodule

validation/tools/pim/pimcomp/compare/run_pimcomp_paper_latency.py

@@ -0,0 +1,755 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import csv
+import json
+import os
+import shlex
+import shutil
+import subprocess
+import sys
+from concurrent.futures import ProcessPoolExecutor, as_completed
+from contextlib import nullcontext, redirect_stderr, redirect_stdout
+from dataclasses import dataclass
+from pathlib import Path
+from tempfile import TemporaryDirectory
+
+from colorama import Fore, Style
+
+
+REPO = Path(__file__).resolve().parents[5]
+SUITE = REPO / "validation/networks/pimcomp_models"
+sys.path.insert(0, str(REPO / "validation"))
+
+from raptor_validation.pimsim_nn import parse_pimsim_nn_metrics  # noqa: E402
+from raptor_validation.validate_one import STAGE_COLORS  # noqa: E402
+
+PIMCOMP_SOURCE = REPO / "third_party/PIMCOMP-NN"
+PIMCOMP_CONFIGS = REPO / "validation/pimsim_configs/pimcomp"
+COMPARE = Path(__file__).resolve().with_name("compare_raptor_pimcomp.py")
+ARCHES = tuple(sorted(path.name for path in PIMCOMP_CONFIGS.iterdir() if path.is_dir()))
+MODELS = {
+    "vgg8": SUITE / "vgg8/vgg8-mnist-reconstructed.onnx",
+    "resnet18": SUITE / "resnet18/resnet18-v1-7.onnx",
+    "resnet34": SUITE / "resnet34/resnet34-v1-7.onnx",
+    "googlenet": SUITE / "googlenet/googlenet-12-pimsim-nn.onnx",
+    "yolo11n": SUITE / "yolo11n/yolo11n-pimsim-nn.onnx",
+}
+COMPARISONS = (
+    ("latency", 1, "element"),
+    ("throughput", 2, "batch"),
+    ("throughput", 4, "batch"),
+    ("throughput", 8, "batch"),
+)
+
+
+@dataclass(frozen=True)
+class ComparisonSpec:
+    label: str
+    model: Path
+    output_dir: Path
+    common_dir: Path
+    config: Path
+    mode: str
+    pipeline: int
+    pimcomp_pipeline: str
+    shared_pimcomp_dir: Path
+    anchor: bool
+
+
+def model_dir(root: Path | None, name: str) -> Path:
+    return root / name if root is not None else MODELS[name].parent
+
+
+def result_dir(root: Path | None, name: str, arch: str, mode: str, pipeline: int) -> Path:
+    base = model_dir(root, name)
+    suffix = "latency" if mode == "latency" else f"throughput/pipeline{pipeline}"
+    return base / arch / suffix
+
+
+def clean_artifacts(root: Path | None, models: list[str], arches: list[str]) -> int:
+    removed = 0
+    for name in models:
+        base = model_dir(root, name)
+        for arch in arches:
+            for path in (base / arch / "latency", base / arch / "throughput"):
+                if path.is_dir() and not path.is_symlink():
+                    shutil.rmtree(path)
+                    removed += 1
+        common = base / "common"
+        if common.is_dir() and not common.is_symlink():
+            shutil.rmtree(common)
+            removed += 1
+    for path in (
+        (root or SUITE) / "results.csv",
+        (root or SUITE) / "results_latency.csv",
+        (root or SUITE) / "results_throughput.csv",
+    ):
+        if path.is_file() or path.is_symlink():
+            path.unlink()
+            removed += 1
+    return removed
+
+
+def write_results_csv(
+    root: Path | None,
+    arch: str,
+    models: list[str],
+    comparisons: tuple[tuple[str, int, str], ...] = COMPARISONS,
+) -> Path:
+    output = (root or SUITE) / "results.csv"
+    fields = (
+        "model",
+        "arch",
+        "mode",
+        "raptor_pipeline",
+        "pimcomp_pipeline",
+        "raptor_functional_validation",
+        "pimcomp_functional_validation",
+        "raptor_throughput_samples_s",
+        "pimcomp_throughput_samples_s",
+        "raptor_latency_ms",
+        "pimcomp_latency_ms",
+        "raptor_power_mw",
+        "pimcomp_power_mw",
+        "raptor_energy_pj",
+        "pimcomp_energy_pj",
+        "better_compiler",
+        "speedup",
+    )
+    selected = {
+        (name, arch, mode, str(pipeline))
+        for name in models
+        for mode, pipeline, _ in comparisons
+    }
+    rows = []
+    if output.exists():
+        with open(output, newline="", encoding="utf-8") as f:
+            reader = csv.DictReader(f)
+            if reader.fieldnames and "model" in reader.fieldnames:
+                for row in reader:
+                    key = (
+                        row.get("model"),
+                        row.get("arch"),
+                        row.get("mode"),
+                        row.get("raptor_pipeline"),
+                    )
+                    if key in selected:
+                        continue
+                    row.pop("status", None)
+                    rows.append({field: row.get(field, "NA") for field in fields})
+    for name in models:
+        for comparison_mode, pipeline, pimcomp_pipeline in comparisons:
+            report_path = result_dir(root, name, arch, comparison_mode, pipeline) / "pimcomp/comparison_report.json"
+            row = {
+                "model": name,
+                "arch": arch,
+                "mode": comparison_mode,
+                "raptor_pipeline": pipeline,
+                "pimcomp_pipeline": pimcomp_pipeline,
+                "raptor_functional_validation": "NA",
+                "pimcomp_functional_validation": "NA",
+                "raptor_throughput_samples_s": "NA",
+                "pimcomp_throughput_samples_s": "NA",
+                "raptor_latency_ms": "NA",
+                "pimcomp_latency_ms": "NA",
+                "raptor_power_mw": "NA",
+                "pimcomp_power_mw": "NA",
+                "raptor_energy_pj": "NA",
+                "pimcomp_energy_pj": "NA",
+                "better_compiler": "NA",
+                "speedup": "NA",
+            }
+            if report_path.exists():
+                report = json.loads(report_path.read_text(encoding="utf-8"))
+                raptor_values = performance_values(report.get("raptor_performance") or {})
+                pimcomp_values = performance_values(report.get("pimcomp_performance") or {})
+                raptor_metric = raptor_values["throughput"] if comparison_mode == "throughput" else raptor_values["latency"]
+                pimcomp_metric = pimcomp_values["throughput"] if comparison_mode == "throughput" else pimcomp_values["latency"]
+                row.update(
+                    raptor_functional_validation=functional_validation_status(
+                        report.get("raptor_validation")
+                    ),
+                    pimcomp_functional_validation=functional_validation_status(
+                        report.get("pimcomp_validation")
+                    ),
+                    raptor_throughput_samples_s=format_value(raptor_values["throughput"]),
+                    pimcomp_throughput_samples_s=format_value(pimcomp_values["throughput"]),
+                    raptor_latency_ms=format_value(raptor_values["latency"]),
+                    pimcomp_latency_ms=format_value(pimcomp_values["latency"]),
+                    raptor_power_mw=format_value(raptor_values["power"]),
+                    pimcomp_power_mw=format_value(pimcomp_values["power"]),
+                    raptor_energy_pj=format_value(raptor_values["energy"]),
+                    pimcomp_energy_pj=format_value(pimcomp_values["energy"]),
+                )
+                if raptor_metric is not None and pimcomp_metric is not None:
+                    row.update(
+                        better_compiler=comparison_winner(comparison_mode, raptor_metric, pimcomp_metric),
+                        speedup=f"{max(raptor_metric, pimcomp_metric) / min(raptor_metric, pimcomp_metric):.2f}",
+                    )
+            rows.append(row)
+    rows.sort(key=lambda row: (list(MODELS).index(row["model"]), row["mode"], int(row["raptor_pipeline"])))
+    with open(output, "w", newline="", encoding="utf-8") as f:
+        writer = csv.DictWriter(f, fieldnames=fields, lineterminator="\n")
+        writer.writeheader()
+        writer.writerows(rows)
+    return output
+
+
+def performance_values(performance: dict) -> dict[str, float | None]:
+    parsed = parse_pimsim_nn_metrics(performance.get("raw_output", ""))
+    return {
+        "throughput": performance.get("throughput") or parsed.get("throughput"),
+        "latency": (
+            performance.get("latency_ms")
+            or performance.get("average_latency_ms")
+            or parsed.get("latency_ms")
+            or parsed.get("average_latency_ms")
+        ),
+        "power": performance.get("average_power_mw") or parsed.get("average_power_mw"),
+        "energy": performance.get("average_energy_pj") or parsed.get("average_energy_pj"),
+    }
+
+
+def comparison_passed(report: dict) -> bool:
+    if report.get("failures"):
+        return False
+    for key in ("raptor_validation", "pimcomp_validation"):
+        result = report.get(key) or {}
+        if result.get("status") != "done" or not result.get("passed"):
+            return False
+    for key in ("raptor_performance", "pimcomp_performance"):
+        performance = report.get(key) or {}
+        if performance.get("error") or performance.get("skipped"):
+            return False
+    return True
+
+
+def functional_validation_status(result: dict | None) -> str:
+    if result is None:
+        return "NA"
+    return "PASS" if result.get("status") == "done" and result.get("passed") else "FAIL"
+
+
+def comparison_winner(mode: str, raptor: float, pimcomp: float) -> str:
+    if raptor == pimcomp:
+        return "tie"
+    if mode == "throughput":
+        return "raptor" if raptor > pimcomp else "pimcomp"
+    return "raptor" if raptor < pimcomp else "pimcomp"
+
+
+def format_value(value: float | None) -> str:
+    return "NA" if value is None else f"{value:.6f}"
+
+
+def print_stage(title: str, color: str) -> None:
+    print("\n" + Style.BRIGHT + color + f"[{title}]" + Style.RESET_ALL, flush=True)
+
+
+def run(command: list[str], *, dry_run: bool, check: bool = True) -> int:
+    print(f"  cwd: {REPO}", flush=True)
+    print(f"  $ {shlex.join(command)}", flush=True)
+    if dry_run:
+        return 0
+    return subprocess.run(command, cwd=REPO, check=check).returncode
+
+
+def run_comparison_job(job: tuple[str, list[str], Path | None]) -> tuple[str, int, str | None]:
+    label, command, log_path = job
+    sys.stdout.flush()
+    sys.stderr.flush()
+    if log_path is None:
+        return label, run(command, dry_run=False, check=False), None
+
+    saved_stdout = os.dup(1)
+    saved_stderr = os.dup(2)
+    try:
+        with open(log_path, "w", encoding="utf-8", buffering=1) as log:
+            os.dup2(log.fileno(), 1)
+            os.dup2(log.fileno(), 2)
+            with redirect_stdout(log), redirect_stderr(log):
+                returncode = run(command, dry_run=False, check=False)
+    finally:
+        os.dup2(saved_stdout, 1)
+        os.dup2(saved_stderr, 2)
+        os.close(saved_stdout)
+        os.close(saved_stderr)
+    return label, returncode, str(log_path)
+
+
+def validate_pimcomp_source() -> None:
+    header = PIMCOMP_SOURCE / "backend/GeneticAlgorithm.h"
+    source = header.read_text(encoding="utf-8")
+    for setting in ("int population_num = 200;", "int max_iteration = 1000;"):
+        if setting not in source:
+            raise RuntimeError(f"PIMCOMP paper setting is missing: {setting}")
+
+
+def comparison_command(
+    model: Path,
+    result_dir: Path,
+    common_dir: Path,
+    config: Path,
+    mode: str,
+    pipeline: int,
+    pimcomp_pipeline: str,
+    pimsim_time_ms: int,
+    timeout: float,
+    reuse_raptor_report: Path | None = None,
+    reuse_pimcomp_dir: Path | None = None,
+) -> list[str]:
+    time_args = ["--pimsim-time-ms", str(pimsim_time_ms)] if mode == "throughput" else []
+    reuse_args = []
+    if reuse_raptor_report is not None:
+        reuse_args.extend(["--reuse-raptor-report", str(reuse_raptor_report)])
+    if reuse_pimcomp_dir is not None:
+        reuse_args.extend(["--reuse-pimcomp-dir", str(reuse_pimcomp_dir)])
+    return [
+        sys.executable,
+        str(COMPARE),
+        "--model",
+        str(model),
+        "--out-dir",
+        str(result_dir),
+        "--common-dir",
+        str(common_dir),
+        "--pimcomp-dir",
+        str(PIMCOMP_SOURCE),
+        "--pimcomp-config",
+        str(config),
+        "--pimsim-mode",
+        mode,
+        *time_args,
+        "--pimcomp-pipeline",
+        pimcomp_pipeline,
+        "--pimcomp-replication",
+        "GA",
+        f"--raptor-extra-arg=--pipeline={pipeline}",
+        "--timeout-seconds",
+        str(timeout),
+        "--fail-on-error",
+        *reuse_args,
+    ]
+
+
+def prepare_common_command(model: Path, common_dir: Path, timeout: float) -> list[str]:
+    return [
+        sys.executable,
+        str(COMPARE),
+        "--model",
+        str(model),
+        "--out-dir",
+        str(common_dir),
+        "--common-dir",
+        str(common_dir),
+        "--prepare-common",
+        "--timeout-seconds",
+        str(timeout),
+    ]
+
+
+def comparison_command_for(
+    spec: ComparisonSpec,
+    args: argparse.Namespace,
+    *,
+    reuse_shared_pimcomp: bool,
+) -> list[str]:
+    reuse_pimcomp_dir = None
+    if reuse_shared_pimcomp:
+        reuse_pimcomp_dir = spec.shared_pimcomp_dir
+    elif args.only == "raptor":
+        reuse_pimcomp_dir = spec.output_dir / "pimcomp/output"
+    return comparison_command(
+        spec.model,
+        spec.output_dir,
+        spec.common_dir,
+        spec.config,
+        spec.mode,
+        spec.pipeline,
+        spec.pimcomp_pipeline,
+        args.pimsim_time_ms,
+        args.timeout_seconds,
+        reuse_raptor_report=(
+            spec.output_dir / "pimcomp/comparison_report.json"
+            if args.only == "pimcomp"
+            else None
+        ),
+        reuse_pimcomp_dir=reuse_pimcomp_dir,
+    )
+
+
+def pimcomp_artifact_ready(output_dir: Path) -> bool:
+    return all(
+        (output_dir / name).is_file()
+        for name in ("SimulationInfo.gz", "VerificationInfo.json", "MappingResult.txt")
+    ) and (output_dir.parent / "comparison_report.json").is_file()
+
+
+def run_comparison_jobs(
+    comparison_jobs: list[tuple[str, list[str], Path | None]],
+    max_workers: int,
+    log_dir: Path | None,
+    log_offset: int,
+) -> tuple[list[str], int]:
+    if not comparison_jobs:
+        return [], log_offset
+    print_directly = min(max_workers, len(comparison_jobs)) == 1
+    jobs = [
+        (
+            label,
+            command,
+            None if print_directly or log_dir is None else log_dir / f"{log_offset + index}.log",
+        )
+        for index, (label, command, _) in enumerate(comparison_jobs)
+    ]
+    with (nullcontext(None) if print_directly else ProcessPoolExecutor(max_workers=max_workers)) as executor:
+        completed = (
+            map(run_comparison_job, jobs)
+            if print_directly
+            else (
+                future.result()
+                for future in as_completed(executor.submit(run_comparison_job, job) for job in jobs)
+            )
+        )
+        failed = []
+        for label, returncode, result_log in completed:
+            if result_log:
+                output = Path(result_log).read_text(encoding="utf-8", errors="replace")
+                if output:
+                    print(output, end="" if output.endswith("\n") else "\n")
+            if returncode:
+                failed.append(label)
+    return failed, log_offset + len(jobs)
+
+
+def ensure_throughput_config(arch: str, sim_time_ms: int) -> Path:
+    source = PIMCOMP_CONFIGS / arch / "throughput_config_1000ms.json"
+    target = PIMCOMP_CONFIGS / arch / f"throughput_config_{sim_time_ms}ms.json"
+    if target.exists():
+        return target
+    if sim_time_ms == 1000:
+        return source
+    config = json.loads(source.read_text(encoding="utf-8"))
+    config["sim_config"]["sim_time"] = sim_time_ms
+    target.write_text(json.dumps(config, indent=2) + "\n", encoding="utf-8")
+    return target
+
+
+def config_path(arch: str, mode: str, sim_time_ms: int, *, write: bool) -> Path:
+    path = (
+        PIMCOMP_CONFIGS / arch / "latency_config.json"
+        if mode == "latency"
+        else PIMCOMP_CONFIGS / arch / f"throughput_config_{sim_time_ms}ms.json"
+    )
+    if mode == "throughput" and write:
+        path = ensure_throughput_config(arch, sim_time_ms)
+    if not path.exists() and (write or mode == "latency"):
+        raise ValueError(f"{arch} has no {mode} config: {path}")
+    return path
+
+
+def core_count(config: Path) -> int:
+    if not config.exists() and config.name.startswith("throughput_config_"):
+        config = config.with_name("throughput_config_1000ms.json")
+    with open(config, encoding="utf-8") as f:
+        return int(json.load(f)["chip_config"]["core_cnt"])
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(
+        description="Compare supported PIMCOMP models with Raptor latency and throughput schedules."
+    )
+    parser.add_argument(
+        "--out-dir",
+        type=Path,
+        help="Result root (default: artifacts beside each model under validation/).",
+    )
+    parser.add_argument("--models", nargs="+", choices=MODELS, default=list(MODELS))
+    parser.add_argument(
+        "--arch",
+        nargs="+",
+        choices=ARCHES,
+        default=list(ARCHES),
+        help="PIM architectures to run (default: all architectures).",
+    )
+    parser.add_argument(
+        "--mode",
+        nargs="+",
+        choices=("latency", "throughput"),
+        default=["latency", "throughput"],
+        help="Pimsim modes to run (default: both).",
+    )
+    parser.add_argument(
+        "--only",
+        choices=("raptor", "pimcomp"),
+        help="Re-run only this compiler and reuse the other side's existing artifacts.",
+    )
+    parser.add_argument(
+        "--pipeline",
+        type=int,
+        choices=(1, 2, 4, 8),
+        help="Run only this Raptor pipeline within the selected mode.",
+    )
+    parser.add_argument(
+        "--pimsim-time-ms",
+        type=int,
+        default=100,
+        help="throughput pimsim-nn horizon in ms (default: 100).",
+    )
+    parser.add_argument(
+        "--timeout-seconds",
+        type=float,
+        default=0.0,
+        help="Per-stage timeout in seconds; 0 means no timeout (default).",
+    )
+    parser.add_argument(
+        "-j",
+        "--jobs",
+        type=int,
+        default=os.cpu_count() or 1,
+        help="Number of comparisons to run in parallel (default: all available CPUs).",
+    )
+    parser.add_argument(
+        "--clean",
+        action="store_true",
+        help="Remove generated comparison artifacts and result summaries, then exit.",
+    )
+    parser.add_argument("--dry-run", action="store_true", help="Print commands without modifying files.")
+    args = parser.parse_args()
+
+    out_dir = args.out_dir.resolve() if args.out_dir is not None else None
+    args.arch = list(dict.fromkeys(args.arch))
+    args.mode = list(dict.fromkeys(args.mode))
+    if args.clean:
+        print(f"Removed {clean_artifacts(out_dir, args.models, args.arch)} comparison artifact path(s).")
+        return 0
+
+    if args.jobs < 1:
+        parser.error("--jobs must be at least 1")
+    if args.pimsim_time_ms <= 0:
+        parser.error("--pimsim-time-ms must be positive")
+    if args.timeout_seconds < 0:
+        parser.error("--timeout-seconds must be non-negative")
+    comparisons_by_arch: dict[str, tuple[tuple[str, int, str], ...]] = {}
+    configs_by_arch: dict[str, dict[str, Path]] = {}
+    for arch in args.arch:
+        comparisons = tuple(
+            comparison for comparison in COMPARISONS
+            if comparison[0] in args.mode
+            and (args.pipeline is None or comparison[1] == args.pipeline)
+        )
+        if not comparisons:
+            parser.error("--pipeline does not belong to the selected --mode")
+        configs = {
+            mode: config_path(arch, mode, args.pimsim_time_ms, write=not args.dry_run)
+            for mode, _, _ in comparisons
+        }
+        unsupported = [
+            pipeline for mode, pipeline, _ in comparisons
+            if mode == "throughput" and core_count(configs[mode]) % pipeline
+        ]
+        if unsupported:
+            if args.pipeline is not None:
+                parser.error(
+                    f"{arch} has {core_count(configs['throughput'])} cores; "
+                    f"throughput pipelines must divide that count (invalid: {unsupported})"
+                )
+            print(
+                Fore.YELLOW
+                + f"Skipping unsupported {arch} throughput pipeline(s): {unsupported}"
+                + Style.RESET_ALL
+            )
+            comparisons = tuple(comparison for comparison in comparisons if comparison[1] not in unsupported)
+        comparisons_by_arch[arch] = comparisons
+        configs_by_arch[arch] = configs
+
+    missing = [str(MODELS[name]) for name in args.models if not MODELS[name].exists()]
+    if missing:
+        parser.error(f"missing model(s): {', '.join(missing)}")
+
+    if args.only is not None:
+        missing_reuse = []
+        for arch in args.arch:
+            for name in args.models:
+                for mode, pipeline, _ in comparisons_by_arch[arch]:
+                    comparison_dir = result_dir(out_dir, name, arch, mode, pipeline)
+                    required = (
+                        comparison_dir / "pimcomp/comparison_report.json"
+                        if args.only == "pimcomp"
+                        else comparison_dir / "pimcomp/output/SimulationInfo.gz"
+                    )
+                    if not required.exists():
+                        missing_reuse.append(str(required))
+                    elif args.only == "raptor":
+                        report = comparison_dir / "pimcomp/comparison_report.json"
+                        if not report.exists() or json.loads(report.read_text(encoding="utf-8")).get(
+                            "pimcomp_model_source"
+                        ) != "original_onnx":
+                            missing_reuse.append(f"{report} (not generated from the original ONNX model)")
+        if missing_reuse:
+            parser.error(
+                f"--only {args.only} needs existing comparison artifacts: "
+                + ", ".join(missing_reuse)
+            )
+
+    validate_pimcomp_source()
+    if out_dir is not None and not args.dry_run:
+        out_dir.mkdir(parents=True, exist_ok=True)
+
+    print(Style.BRIGHT + f"Found {len(args.models)} PIMCOMP model(s) to compare." + Style.RESET_ALL)
+    print(f"Architectures: {', '.join(args.arch)}")
+    print(f"Modes: {', '.join(args.mode)}")
+    print(f"Throughput pimsim time: {args.pimsim_time_ms} ms")
+    print(f"Max parallel jobs: {args.jobs}")
+    print(f"Results root: {out_dir or SUITE}")
+    print("=" * 72)
+
+    print_stage("Build Raptor", STAGE_COLORS["Build Runner"])
+    run(["cmake", "--build", str(REPO / "build_release")], dry_run=args.dry_run)
+
+    if args.only != "raptor":
+        print_stage("Build PIMCOMP", STAGE_COLORS["Build Runner"])
+        run(
+            ["cmake", "--build", str(PIMCOMP_SOURCE / "build"), "--target", "PIMCOMP-NN"],
+            dry_run=args.dry_run,
+        )
+
+    print_stage("Prepare shared artifacts", STAGE_COLORS["Build Runner"])
+    for name in args.models:
+        run(
+            prepare_common_command(
+                MODELS[name],
+                model_dir(out_dir, name) / "common",
+                args.timeout_seconds,
+            ),
+            dry_run=args.dry_run,
+        )
+
+    failed = []
+    comparison_specs: list[ComparisonSpec] = []
+    shared_pimcomp_by_group: dict[tuple[str, str, str], Path] = {}
+    for arch in args.arch:
+        comparisons = comparisons_by_arch[arch]
+        configs = configs_by_arch[arch]
+        for index, name in enumerate(args.models, start=1):
+            for mode, pipeline, pimcomp_pipeline in comparisons:
+                model_result_dir = result_dir(out_dir, name, arch, mode, pipeline)
+                label = f"{arch}/{name}/{mode}/pipeline{pipeline}"
+                group = (arch, name, mode)
+                shared_pimcomp_dir = shared_pimcomp_by_group.get(group)
+                anchor = shared_pimcomp_dir is None
+                if anchor:
+                    shared_pimcomp_dir = model_result_dir / "pimcomp/output"
+                    shared_pimcomp_by_group[group] = shared_pimcomp_dir
+                print(
+                    "\n" + Fore.CYAN + f"[{arch} {index}/{len(args.models)}]" + Style.RESET_ALL
+                    + f" {Style.BRIGHT}Comparing {name} ({mode}, pipeline={pipeline}){Style.RESET_ALL}",
+                    flush=True,
+                )
+                comparison_specs.append(
+                    ComparisonSpec(
+                        label=label,
+                        model=MODELS[name],
+                        output_dir=model_result_dir,
+                        common_dir=model_dir(out_dir, name) / "common",
+                        config=configs[mode],
+                        mode=mode,
+                        pipeline=pipeline,
+                        pimcomp_pipeline=pimcomp_pipeline,
+                        shared_pimcomp_dir=shared_pimcomp_dir,
+                        anchor=anchor,
+                    )
+                )
+
+    if args.dry_run:
+        for spec in comparison_specs:
+            command = comparison_command_for(
+                spec,
+                args,
+                reuse_shared_pimcomp=not spec.anchor,
+            )
+            if run(command, dry_run=True, check=False):
+                failed.append(spec.label)
+    elif comparison_specs:
+        print(f"Running {len(comparison_specs)} comparison(s)")
+        anchor_jobs = [
+            (
+                spec.label,
+                comparison_command_for(spec, args, reuse_shared_pimcomp=False),
+                None,
+            )
+            for spec in comparison_specs
+            if spec.anchor
+        ]
+        dependent_specs = [spec for spec in comparison_specs if not spec.anchor]
+        print_directly = min(args.jobs, len(comparison_specs)) == 1
+        with (nullcontext(None) if print_directly else TemporaryDirectory(prefix="raptor-pimcomp-")) as log_dir:
+            anchor_failed, log_offset = run_comparison_jobs(
+                anchor_jobs,
+                args.jobs,
+                Path(log_dir) if log_dir else None,
+                0,
+            )
+            failed.extend(anchor_failed)
+            dependent_jobs = [
+                (
+                    spec.label,
+                    comparison_command_for(
+                        spec,
+                        args,
+                        reuse_shared_pimcomp=pimcomp_artifact_ready(spec.shared_pimcomp_dir),
+                    ),
+                    None,
+                )
+                for spec in dependent_specs
+            ]
+            dependent_failed, _ = run_comparison_jobs(
+                dependent_jobs,
+                args.jobs,
+                Path(log_dir) if log_dir else None,
+                log_offset,
+            )
+            failed.extend(dependent_failed)
+
+    if args.dry_run:
+        return 1 if failed else 0
+
+    results_path = None
+    for arch in args.arch:
+        results_path = write_results_csv(
+            out_dir,
+            arch,
+            args.models,
+            comparisons_by_arch[arch],
+        )
+    assert results_path is not None
+    print_stage(results_path.name, STAGE_COLORS["Compare Outputs"])
+    print(results_path.read_text(encoding="utf-8"), end="")
+    for arch in args.arch:
+        for name in args.models:
+            for mode, pipeline, _ in comparisons_by_arch[arch]:
+                label = f"{arch}/{name}/{mode}/pipeline{pipeline}"
+                report_path = result_dir(out_dir, name, arch, mode, pipeline) / "pimcomp/comparison_report.json"
+                if not report_path.exists() or not comparison_passed(
+                    json.loads(report_path.read_text(encoding="utf-8"))
+                ):
+                    if label not in failed:
+                        failed.append(label)
+    print("\n" + Style.BRIGHT + Fore.CYAN + "Summary" + Style.RESET_ALL)
+    total_jobs = sum(len(args.models) * len(comparisons) for comparisons in comparisons_by_arch.values())
+    print(Style.BRIGHT + f"Passed: {total_jobs - len(failed)}" + Style.RESET_ALL)
+    print(Style.BRIGHT + f"Failed: {len(failed)}" + Style.RESET_ALL)
+    print(Style.BRIGHT + f"Results: {results_path}" + Style.RESET_ALL)
+    if failed:
+        print(
+            Fore.RED + f"Failed comparisons: {', '.join(failed)}" + Style.RESET_ALL,
+            file=sys.stderr,
+        )
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())