better comparison scripts

2026-08-07 13:19:03 +02:00
parent ac84040e16
commit 1b7d22b87e
2 changed files with 187 additions and 125 deletions
@@ -145,8 +145,13 @@ def run_logged(
    timeout_sec: float,
    steps: list[StepRecord],
    stage: str | None = None,
+    print_header: bool = True,
 ) -> str:
-    print_step(name, cmd, cwd, stage)
+    if print_header:
+        print_step(name, cmd, cwd, stage)
+    else:
+        print(f"  cwd: {cwd or REPO}")
+        print(f"  $ {shell_join(cmd)}")
    start = time.perf_counter()
    command = shell_join(cmd)
    try:
@@ -446,7 +451,7 @@ def compile_raptor_target(
        "--pim-emit-json",
        *args.raptor_extra_arg,
    ]
-    print_step("Compile Raptor PIM", cmd, REPO, "Compile PIM")
+    print_step("Compile Raptor", cmd, REPO, "Compile PIM")
    start = time.perf_counter()
    command = shell_join(cmd)
    raptor_extra_args = [
@@ -566,6 +571,7 @@ def compile_pimcomp(
        (pimcomp_output_dir / name).unlink(missing_ok=True)
    model_name = args.pimcomp_model_name or f"compare_{model_path.stem}"
    frontend_json = frontend_json_dir / f"{model_name}.json"
+    print_step("Compile PIMCOMP", stage="Compile PIM")
    # The original PIMCOMP frontend rewrites its input ONNX while loading it.
    # Isolate that mutation without changing the model sent through PIMCOMP.
    with TemporaryDirectory(prefix="pimcomp-model-") as temp_dir:
@@ -586,6 +592,7 @@ def compile_pimcomp(
            timeout_sec=args.timeout_seconds,
            steps=steps,
            stage="Compile PIM",
+            print_header=False,
        )
    backend_cmd = [
        str(args.pimcomp_dir / "build" / "PIMCOMP-NN"),
@@ -603,6 +610,7 @@ def compile_pimcomp(
            timeout_sec=args.timeout_seconds,
            steps=steps,
            stage="Compile PIM",
+            print_header=False,
        )
    finally:
        shutil.rmtree(frontend_json_dir.parent, ignore_errors=True)
@@ -1014,6 +1022,28 @@ def optional_path(path: Path | None) -> str | None:
    return str(path) if path is not None else None


+def path_from_report(report: dict[str, Any], name: str) -> Path | None:
+    value = report.get("paths", {}).get(name)
+    return Path(value) if value else None
+
+
+def restore_side_records(
+    report: dict[str, Any],
+    side: str,
+    failures: list[dict[str, str]],
+    steps: list[StepRecord],
+) -> None:
+    marker = side.upper()
+    failures.extend(
+        failure for failure in report.get("failures", [])
+        if marker in failure.get("stage", "").upper()
+    )
+    steps.extend(
+        StepRecord(**step) for step in report.get("steps", [])
+        if marker in step.get("name", "").upper()
+    )
+
+
 def default_common_dir(out_dir: Path) -> Path:
    if out_dir.name == "latency":
        return out_dir.parents[1] / "common"
@@ -1280,6 +1310,11 @@ def main():
        type=Path,
        help="Reuse a directory containing PIMCOMP SimulationInfo.gz, VerificationInfo.json, and MappingResult.txt.",
    )
+    parser.add_argument(
+        "--reuse-pimcomp-report",
+        type=Path,
+        help="Preserve the PIMCOMP side of an existing comparison report without rerunning it.",
+    )
    parser.add_argument("--skip-pimsim-nn", action="store_true")
    parser.add_argument("--verbose-raptor-compile", action="store_true")
    parser.add_argument("--raptor-extra-arg", action="append", default=[])
@@ -1289,6 +1324,8 @@ def main():
        help="Return a non-zero status if a stage or semantic validation fails.",
    )
    args = parser.parse_args()
+    if args.reuse_pimcomp_dir is not None and args.reuse_pimcomp_report is not None:
+        parser.error("--reuse-pimcomp-dir and --reuse-pimcomp-report are mutually exclusive")
    if args.pimsim_time_ms <= 0:
        parser.error("--pimsim-time-ms must be positive")
    if args.timeout_seconds < 0:
@@ -1340,6 +1377,7 @@ def main():
    verification_info: Path | None = None
    simulation_info: Path | None = None
    pimcomp_export_dir: Path | None = None
+    pimcomp_pimsim_dir: Path | None = None
    pimcomp_model_path: Path | None = None
    pimsim_config: Path | None = None
    reuse_raptor = args.reuse_raptor_report is not None
@@ -1360,11 +1398,14 @@ def main():
        inputs_desc, outputs_desc, arrays_in_order = model_io
        runtime_inputs = arrays_in_order

-    reuse_pimcomp = args.reuse_pimcomp_dir is not None
-    if reuse_pimcomp:
+    reused_pimcomp_report_path = args.reuse_pimcomp_report
+    if args.reuse_pimcomp_dir is not None:
        reused_pimcomp_report_path = args.reuse_pimcomp_dir.resolve().parent / "comparison_report.json"
+    reuse_pimcomp = reused_pimcomp_report_path is not None
+    if reuse_pimcomp:
+        reused_pimcomp_report_path = reused_pimcomp_report_path.resolve()
        if not reused_pimcomp_report_path.exists():
-            raise ValueError(f"Missing PIMCOMP report beside {args.reuse_pimcomp_dir}")
+            raise ValueError(f"Missing PIMCOMP report: {reused_pimcomp_report_path}")
        with open(reused_pimcomp_report_path, "r", encoding="utf-8") as f:
            reused_pimcomp = json.load(f)
        if reused_pimcomp.get("pimcomp_model_source") != "original_onnx":
@@ -1372,9 +1413,13 @@ def main():
        pimcomp_validation = CompareResult(**reused_pimcomp["pimcomp_validation"])
        pimcomp_perf = reused_pimcomp["pimcomp_performance"]
        pimcomp_instr = reused_pimcomp["pimcomp_instruction_summary"]
-        reused_config = reused_pimcomp.get("paths", {}).get("pimsim_config")
-        if reused_config:
-            pimsim_config = Path(reused_config)
+        simulation_info = path_from_report(reused_pimcomp, "pimcomp_simulation_info")
+        pimcomp_export_dir = path_from_report(reused_pimcomp, "pimcomp_exported_pim")
+        pimcomp_pimsim_dir = path_from_report(reused_pimcomp, "pimcomp_pimsim_nn")
+        if pimcomp_pimsim_dir is None and (reused_pimcomp_report_path.parent / "pimsim_nn").is_dir():
+            pimcomp_pimsim_dir = reused_pimcomp_report_path.parent / "pimsim_nn"
+        pimsim_config = path_from_report(reused_pimcomp, "pimsim_config")
+        restore_side_records(reused_pimcomp, "pimcomp", failures, steps)

    if reuse_raptor and model_io is not None:
        reuse_report_path = args.reuse_raptor_report.resolve()
@@ -1384,8 +1429,9 @@ def main():
        if reused_hardware != hardware:
            raise ValueError(f"Reused Raptor hardware differs: {reused_hardware} != {hardware}")
        reference_dir = common_dir / "outputs"
-        reused_raptor_pim = reused["paths"].get("raptor_pim")
-        raptor_pim_dir = Path(reused_raptor_pim) if reused_raptor_pim else None
+        runner_path = path_from_report(reused, "reference_runner")
+        raptor_pim_dir = path_from_report(reused, "raptor_pim")
+        raptor_pimsim_dir = path_from_report(reused, "raptor_pimsim_nn")
        arrays_in_order = load_saved_inputs(
            inputs_desc,
            common_dir / "inputs",
@@ -1395,6 +1441,7 @@ def main():
        raptor_perf = reused["raptor_performance"]
        raptor_instr = reused["raptor_instruction_summary"]
        raptor_pass_timings = reused["raptor_pass_timings"]
+        restore_side_records(reused, "raptor", failures, steps)
        print_step("Reuse Raptor")
        print(f"  Report: {reuse_report_path}")

@@ -1528,7 +1575,7 @@ def main():
            simulation_info = out_dir / "pimcomp/output/SimulationInfo.gz"
            print_step("Reuse PIMCOMP")
            print(f"  Directory: {reused_pimcomp_dir}")
-    else:
+    elif not reuse_pimcomp:
        compiled_pimcomp = try_stage(
            failures,
            "Compile PIMCOMP",
@@ -1614,11 +1661,13 @@ def main():
    if args.skip_pimsim_nn:
        if not reuse_raptor:
            raptor_perf = skipped_perf("Skipped by --skip-pimsim-nn")
-        pimcomp_perf = skipped_perf("Skipped by --skip-pimsim-nn")
+        if not reuse_pimcomp:
+            pimcomp_perf = skipped_perf("Skipped by --skip-pimsim-nn")
    elif pimsim_config is None:
        if not reuse_raptor:
            raptor_perf = skipped_perf("pimsim-nn config is not available")
-        pimcomp_perf = skipped_perf("pimsim-nn config is not available")
+        if not reuse_pimcomp:
+            pimcomp_perf = skipped_perf("pimsim-nn config is not available")
    else:
        if not reuse_raptor and raptor_pim_dir is not None:
            raptor_pimsim_dir = try_stage(
@@ -1645,30 +1694,31 @@ def main():
        elif not reuse_raptor:
            raptor_perf = skipped_perf("Raptor PIM directory is not available")

-        if not reuse_pimcomp and simulation_info is not None:
-            pimcomp_pimsim_dir = try_stage(
-                failures,
-                "Export PIMCOMP for pimsim-nn",
-                export_pimcomp_for_pimsim_nn,
-                simulation_info,
-                out_dir / "pimcomp/pimsim_nn",
-            )
-            if pimcomp_pimsim_dir is not None:
-                perf = try_stage(
+        if not reuse_pimcomp:
+            if simulation_info is not None:
+                pimcomp_pimsim_dir = try_stage(
                    failures,
-                    "Non-Functional Simulation PIMCOMP",
-                    run_pimsim_nn,
-                    "Non-Functional Simulation PIMCOMP",
-                    pimcomp_pimsim_dir,
-                    pimsim_config,
-                    steps,
-                    args,
+                    "Export PIMCOMP for pimsim-nn",
+                    export_pimcomp_for_pimsim_nn,
+                    simulation_info,
+                    out_dir / "pimcomp/pimsim_nn",
                )
-                pimcomp_perf = perf if perf is not None else failed_perf("pimsim-nn PIMCOMP failed")
+                if pimcomp_pimsim_dir is not None:
+                    perf = try_stage(
+                        failures,
+                        "Non-Functional Simulation PIMCOMP",
+                        run_pimsim_nn,
+                        "Non-Functional Simulation PIMCOMP",
+                        pimcomp_pimsim_dir,
+                        pimsim_config,
+                        steps,
+                        args,
+                    )
+                    pimcomp_perf = perf if perf is not None else failed_perf("pimsim-nn PIMCOMP failed")
+                else:
+                    pimcomp_perf = failed_perf("PIMCOMP pimsim-nn export failed")
            else:
-                pimcomp_perf = failed_perf("PIMCOMP pimsim-nn export failed")
-        else:
-            pimcomp_perf = skipped_perf("PIMCOMP SimulationInfo.gz is not available")
+                pimcomp_perf = skipped_perf("PIMCOMP SimulationInfo.gz is not available")

    if not reuse_raptor and raptor_pim_dir is not None and raptor_pim_dir.exists():
        parsed = try_stage(failures, "Parse Raptor instructions", parse_raptor_instructions, raptor_pim_dir)
@@ -1676,11 +1726,12 @@ def main():
    elif not reuse_raptor:
        raptor_instr = empty_instruction_summary("Raptor PIM directory is not available")

-    if not reuse_pimcomp and simulation_info is not None and simulation_info.exists():
-        parsed = try_stage(failures, "Parse PIMCOMP instructions", parse_pimcomp_instructions, simulation_info)
-        pimcomp_instr = parsed if parsed is not None else empty_instruction_summary(error="Failed to parse PIMCOMP instructions")
-    else:
-        pimcomp_instr = empty_instruction_summary("PIMCOMP SimulationInfo.gz is not available")
+    if not reuse_pimcomp:
+        if simulation_info is not None and simulation_info.exists():
+            parsed = try_stage(failures, "Parse PIMCOMP instructions", parse_pimcomp_instructions, simulation_info)
+            pimcomp_instr = parsed if parsed is not None else empty_instruction_summary(error="Failed to parse PIMCOMP instructions")
+        else:
+            pimcomp_instr = empty_instruction_summary("PIMCOMP SimulationInfo.gz is not available")

    report_path = out_dir / "pimcomp/comparison_report.md"
    write_report(
@@ -1712,6 +1763,7 @@ def main():
        "pimcomp_config": str(args.pimcomp_config),
        "raptor_extra_args": args.raptor_extra_arg,
        "reused_raptor_report": optional_path(args.reuse_raptor_report.resolve()) if reuse_raptor else None,
+        "reused_pimcomp_report": optional_path(reused_pimcomp_report_path) if reuse_pimcomp else None,
        "failures": failures,
        "steps": [asdict(step) for step in steps],
        "raptor_validation": asdict(raptor_validation),
@@ -1730,6 +1782,7 @@ def main():
            "raptor_pimsim_nn": optional_path(raptor_pimsim_dir),
            "pimcomp_simulation_info": optional_path(simulation_info),
            "pimcomp_exported_pim": optional_path(pimcomp_export_dir),
+            "pimcomp_pimsim_nn": optional_path(pimcomp_pimsim_dir),
            "pimsim_config": optional_path(pimsim_config),
            "report_markdown": str(report_path),
        },
@@ -1739,15 +1792,28 @@ def main():
        json.dump(json_report, f, indent=2)
        f.write("\n")

-    failed_steps = any(step.status != "passed" for step in steps)
+    partial_side = "pimcomp" if reuse_raptor else "raptor" if args.reuse_pimcomp_report else None
+    other_side = "RAPTOR" if partial_side == "pimcomp" else "PIMCOMP"
+    relevant_failures = (
+        failures if partial_side is None
+        else [failure for failure in failures if other_side not in failure["stage"].upper()]
+    )
+    relevant_steps = (
+        steps if partial_side is None
+        else [step for step in steps if other_side not in step.name.upper()]
+    )
+    relevant_validations = (
+        (raptor_validation, pimcomp_validation)
+        if partial_side is None
+        else (raptor_validation,) if partial_side == "raptor" else (pimcomp_validation,)
+    )
+    failed_steps = any(step.status != "passed" for step in relevant_steps)
    functional_failure = any(
        result.status == "done" and not result.passed
-        for result in (raptor_validation, pimcomp_validation)
+        for result in relevant_validations
    )
-    failed = bool(failures or failed_steps or functional_failure)
-    result = "FAIL" if args.fail_on_error and failed else "DONE" if failed else "PASS"
-    color = Fore.RED if result == "FAIL" else Fore.YELLOW if result == "DONE" else Fore.GREEN
-    print("\n" + Style.BRIGHT + f"Result: {color}{result}" + Style.RESET_ALL)
+    failed = bool(relevant_failures or failed_steps or functional_failure)
+    print("\n" + Style.BRIGHT + Fore.GREEN + "[Completed]" + Style.RESET_ALL)
    print(f"  Report: {report_path}")
    print(f"  JSON:   {json_path}")
    if failures or failed_steps:
@@ -101,8 +101,8 @@ def write_results_csv(
 ) -> Path:
    output = (root or SUITE) / "results.csv"
    fields = (
-        "model",
        "arch",
+        "model",
        "mode",
        "raptor_pipeline",
        "pimcomp_pipeline",
@@ -185,12 +185,26 @@ def write_results_csv(
                    pimcomp_energy_pj=format_value(pimcomp_values["energy"]),
                )
                if raptor_metric is not None and pimcomp_metric is not None:
+                    min_metric = min(raptor_metric, pimcomp_metric)
                    row.update(
                        better_compiler=comparison_winner(comparison_mode, raptor_metric, pimcomp_metric),
-                        speedup=f"{max(raptor_metric, pimcomp_metric) / min(raptor_metric, pimcomp_metric):.2f}",
+                        speedup=(
+                            f"{max(raptor_metric, pimcomp_metric) / min_metric:.2f}"
+                            if min_metric > 0
+                            else "NA"
+                        ),
                    )
            rows.append(row)
-    rows.sort(key=lambda row: (list(MODELS).index(row["model"]), row["mode"], int(row["raptor_pipeline"])))
+    model_order = {name: index for index, name in enumerate(MODELS)}
+    mode_order = {"latency": 0, "throughput": 1}
+    rows.sort(
+        key=lambda row: (
+            row["arch"],
+            model_order[row["model"]],
+            mode_order[row["mode"]],
+            int(row["raptor_pipeline"]),
+        )
+    )
    with open(output, "w", newline="", encoding="utf-8") as f:
        writer = csv.DictWriter(f, fieldnames=fields, lineterminator="\n")
        writer.writeheader()
@@ -213,15 +227,20 @@ def performance_values(performance: dict) -> dict[str, float | None]:
    }


-def comparison_passed(report: dict) -> bool:
-    if report.get("failures"):
+def comparison_passed(report: dict, compiler: str | None = None) -> bool:
+    other_compiler = "PIMCOMP" if compiler == "raptor" else "RAPTOR"
+    if any(
+        compiler is None or other_compiler not in failure.get("stage", "").upper()
+        for failure in report.get("failures", [])
+    ):
        return False
-    for key in ("raptor_validation", "pimcomp_validation"):
-        result = report.get(key) or {}
+    compilers = (compiler,) if compiler is not None else ("raptor", "pimcomp")
+    for name in compilers:
+        result = report.get(f"{name}_validation") or {}
        if result.get("status") != "done" or not result.get("passed"):
            return False
-    for key in ("raptor_performance", "pimcomp_performance"):
-        performance = report.get(key) or {}
+    for name in compilers:
+        performance = report.get(f"{name}_performance") or {}
        if performance.get("error") or performance.get("skipped"):
            return False
    return True
@@ -249,6 +268,13 @@ def print_stage(title: str, color: str) -> None:
    print("\n" + Style.BRIGHT + color + f"[{title}]" + Style.RESET_ALL, flush=True)


+def print_completed(label: str, output: str = "") -> None:
+    print("\n" + Style.BRIGHT + Fore.CYAN + f"[Completed {label}]" + Style.RESET_ALL)
+    if output:
+        print(output, end="" if output.endswith("\n") else "\n")
+    print("=" * 72, flush=True)
+
+
 def run(command: list[str], *, dry_run: bool, check: bool = True) -> int:
    print(f"  cwd: {REPO}", flush=True)
    print(f"  $ {shlex.join(command)}", flush=True)
@@ -277,7 +303,7 @@ def run_comparison_job(job: tuple[str, list[str], Path | None]) -> tuple[str, in
        os.dup2(saved_stderr, 2)
        os.close(saved_stdout)
        os.close(saved_stderr)
-    return label, returncode, str(log_path)
+    return label, returncode, log_path.read_text(encoding="utf-8", errors="replace")


 def validate_pimcomp_source() -> None:
@@ -300,6 +326,7 @@ def comparison_command(
    timeout: float,
    reuse_raptor_report: Path | None = None,
    reuse_pimcomp_dir: Path | None = None,
+    reuse_pimcomp_report: Path | None = None,
 ) -> list[str]:
    time_args = ["--pimsim-time-ms", str(pimsim_time_ms)] if mode == "throughput" else []
    reuse_args = []
@@ -307,6 +334,8 @@ def comparison_command(
        reuse_args.extend(["--reuse-raptor-report", str(reuse_raptor_report)])
    if reuse_pimcomp_dir is not None:
        reuse_args.extend(["--reuse-pimcomp-dir", str(reuse_pimcomp_dir)])
+    if reuse_pimcomp_report is not None:
+        reuse_args.extend(["--reuse-pimcomp-report", str(reuse_pimcomp_report)])
    return [
        sys.executable,
        str(COMPARE),
@@ -357,11 +386,7 @@ def comparison_command_for(
    *,
    reuse_shared_pimcomp: bool,
 ) -> list[str]:
-    reuse_pimcomp_dir = None
-    if reuse_shared_pimcomp:
-        reuse_pimcomp_dir = spec.shared_pimcomp_dir
-    elif args.only == "raptor":
-        reuse_pimcomp_dir = existing_pimcomp_dir(spec.output_dir)
+    report = spec.output_dir / "pimcomp/comparison_report.json"
    return comparison_command(
        spec.model,
        spec.output_dir,
@@ -373,11 +398,12 @@ def comparison_command_for(
        args.pimsim_time_ms,
        args.timeout_seconds,
        reuse_raptor_report=(
-            spec.output_dir / "pimcomp/comparison_report.json"
+            report
            if args.only == "pimcomp"
            else None
        ),
-        reuse_pimcomp_dir=reuse_pimcomp_dir,
+        reuse_pimcomp_dir=spec.shared_pimcomp_dir if reuse_shared_pimcomp and args.only != "raptor" else None,
+        reuse_pimcomp_report=report if args.only == "raptor" else None,
    )


@@ -388,11 +414,6 @@ def pimcomp_artifact_ready(output_dir: Path) -> bool:
    ) and (output_dir.parent / "comparison_report.json").is_file()


-def existing_pimcomp_dir(comparison_dir: Path) -> Path | None:
-    output_dir = comparison_dir / "pimcomp/output"
-    return output_dir if pimcomp_artifact_ready(output_dir) else None
-
-
 def run_comparison_jobs(
    comparison_jobs: list[tuple[str, list[str], Path | None]],
    max_workers: int,
@@ -420,11 +441,8 @@ def run_comparison_jobs(
            )
        )
        failed = []
-        for label, returncode, result_log in completed:
-            if result_log:
-                output = Path(result_log).read_text(encoding="utf-8", errors="replace")
-                if output:
-                    print(output, end="" if output.endswith("\n") else "\n")
+        for label, returncode, output in completed:
+            print_completed(label, output or "")
            if returncode:
                failed.append(label)
    return failed, log_offset + len(jobs)
@@ -456,13 +474,6 @@ def config_path(arch: str, mode: str, sim_time_ms: int, *, write: bool) -> Path:
    return path


-def core_count(config: Path) -> int:
-    if not config.exists() and config.name.startswith("throughput_config_"):
-        config = config.with_name("throughput_config_1000ms.json")
-    with open(config, encoding="utf-8") as f:
-        return int(json.load(f)["chip_config"]["core_cnt"])
-
-
 def main() -> int:
    parser = argparse.ArgumentParser(
        description="Compare supported PIMCOMP models with Raptor latency and throughput schedules."
@@ -490,7 +501,7 @@ def main() -> int:
    parser.add_argument(
        "--only",
        choices=("raptor", "pimcomp"),
-        help="Re-run only this compiler and reuse the other side's existing artifacts.",
+        help="Re-run only this compiler's compile, validation, and simulation stages; preserve the other side from its report.",
    )
    parser.add_argument(
        "--pipeline",
@@ -552,22 +563,6 @@ def main() -> int:
            mode: config_path(arch, mode, args.pimsim_time_ms, write=not args.dry_run)
            for mode, _, _ in comparisons
        }
-        unsupported = [
-            pipeline for mode, pipeline, _ in comparisons
-            if mode == "throughput" and core_count(configs[mode]) % pipeline
-        ]
-        if unsupported:
-            if args.pipeline is not None:
-                parser.error(
-                    f"{arch} has {core_count(configs['throughput'])} cores; "
-                    f"throughput pipelines must divide that count (invalid: {unsupported})"
-                )
-            print(
-                Fore.YELLOW
-                + f"Skipping unsupported {arch} throughput pipeline(s): {unsupported}"
-                + Style.RESET_ALL
-            )
-            comparisons = tuple(comparison for comparison in comparisons if comparison[1] not in unsupported)
        comparisons_by_arch[arch] = comparisons
        configs_by_arch[arch] = configs

@@ -581,19 +576,14 @@ def main() -> int:
            for name in args.models:
                for mode, pipeline, _ in comparisons_by_arch[arch]:
                    comparison_dir = result_dir(out_dir, name, arch, mode, pipeline)
-                    required = (
-                        comparison_dir / "pimcomp/comparison_report.json"
-                        if args.only == "pimcomp"
-                        else existing_pimcomp_dir(comparison_dir)
-                    )
-                    if required is None or not required.exists():
-                        missing_reuse.append(str(required or comparison_dir / "pimcomp"))
+                    required = comparison_dir / "pimcomp/comparison_report.json"
+                    if not required.exists():
+                        missing_reuse.append(str(required))
                    elif args.only == "raptor":
-                        report = comparison_dir / "pimcomp/comparison_report.json"
-                        if not report.exists() or json.loads(report.read_text(encoding="utf-8")).get(
+                        if json.loads(required.read_text(encoding="utf-8")).get(
                            "pimcomp_model_source"
                        ) != "original_onnx":
-                            missing_reuse.append(f"{report} (not generated from the original ONNX model)")
+                            missing_reuse.append(f"{required} (not generated from the original ONNX model)")
        if missing_reuse:
            parser.error(
                f"--only {args.only} needs existing comparison artifacts: "
@@ -609,19 +599,26 @@ def main() -> int:
    print(f"Modes: {', '.join(args.mode)}")
    print(f"Throughput pimsim time: {args.pimsim_time_ms} ms")
    print(f"Max parallel jobs: {args.jobs}")
+    print(
+        f"Comparison jobs: "
+        f"{sum(len(args.models) * len(comparisons) for comparisons in comparisons_by_arch.values())}"
+    )
    print(f"Results root: {out_dir or SUITE}")
    print("=" * 72)

    print_stage("Prepare shared artifacts", STAGE_COLORS["Build Runner"])
    for name in args.models:
-        run(
-            prepare_common_command(
-                MODELS[name],
-                model_dir(out_dir, name) / "common",
-                args.timeout_seconds,
-            ),
-            dry_run=args.dry_run,
-        )
+        try:
+            run(
+                prepare_common_command(
+                    MODELS[name],
+                    model_dir(out_dir, name) / "common",
+                    args.timeout_seconds,
+                ),
+                dry_run=args.dry_run,
+            )
+        finally:
+            print_completed(f"shared artifacts: {name}")

    failed = []
    comparison_specs: list[ComparisonSpec] = []
@@ -629,7 +626,7 @@ def main() -> int:
    for arch in args.arch:
        comparisons = comparisons_by_arch[arch]
        configs = configs_by_arch[arch]
-        for index, name in enumerate(args.models, start=1):
+        for name in args.models:
            for mode, pipeline, pimcomp_pipeline in comparisons:
                model_result_dir = result_dir(out_dir, name, arch, mode, pipeline)
                label = f"{arch}/{name}/{mode}/pipeline{pipeline}"
@@ -639,11 +636,6 @@ def main() -> int:
                if anchor:
                    shared_pimcomp_dir = model_result_dir / "pimcomp/output"
                    shared_pimcomp_by_group[group] = shared_pimcomp_dir
-                print(
-                    "\n" + Fore.CYAN + f"[{arch} {index}/{len(args.models)}]" + Style.RESET_ALL
-                    + f" {Style.BRIGHT}Comparing {name} ({mode}, pipeline={pipeline}){Style.RESET_ALL}",
-                    flush=True,
-                )
                comparison_specs.append(
                    ComparisonSpec(
                        label=label,
@@ -669,17 +661,20 @@ def main() -> int:
            if run(command, dry_run=True, check=False):
                failed.append(spec.label)
    elif comparison_specs:
-        print(f"Running {len(comparison_specs)} comparison(s)")
+        anchor_specs = comparison_specs if args.only == "raptor" else [
+            spec for spec in comparison_specs if spec.anchor
+        ]
        anchor_jobs = [
            (
                spec.label,
                comparison_command_for(spec, args, reuse_shared_pimcomp=False),
                None,
            )
-            for spec in comparison_specs
-            if spec.anchor
+            for spec in anchor_specs
+        ]
+        dependent_specs = [] if args.only == "raptor" else [
+            spec for spec in comparison_specs if not spec.anchor
        ]
-        dependent_specs = [spec for spec in comparison_specs if not spec.anchor]
        print_directly = min(args.jobs, len(comparison_specs)) == 1
        with (nullcontext(None) if print_directly else TemporaryDirectory(prefix="raptor-pimcomp-")) as log_dir:
            anchor_failed, log_offset = run_comparison_jobs(
@@ -729,7 +724,8 @@ def main() -> int:
                label = f"{arch}/{name}/{mode}/pipeline{pipeline}"
                report_path = result_dir(out_dir, name, arch, mode, pipeline) / "pimcomp/comparison_report.json"
                if not report_path.exists() or not comparison_passed(
-                    json.loads(report_path.read_text(encoding="utf-8"))
+                    json.loads(report_path.read_text(encoding="utf-8")),
+                    args.only,
                ):
                    if label not in failed:
                        failed.append(label)