better comparison scripts

validation/tools/pim/pimcomp/compare/compare_raptor_pimcomp.py

@@ -145,8 +145,13 @@ def run_logged(
     timeout_sec: float,
     steps: list[StepRecord],
     stage: str | None = None,
+    print_header: bool = True,
 ) -> str:
+    if print_header:
         print_step(name, cmd, cwd, stage)
+    else:
+        print(f"  cwd: {cwd or REPO}")
+        print(f"  $ {shell_join(cmd)}")
     start = time.perf_counter()
     command = shell_join(cmd)
     try:
@@ -446,7 +451,7 @@ def compile_raptor_target(
         "--pim-emit-json",
         *args.raptor_extra_arg,
     ]
-    print_step("Compile Raptor PIM", cmd, REPO, "Compile PIM")
+    print_step("Compile Raptor", cmd, REPO, "Compile PIM")
     start = time.perf_counter()
     command = shell_join(cmd)
     raptor_extra_args = [
@@ -566,6 +571,7 @@ def compile_pimcomp(
         (pimcomp_output_dir / name).unlink(missing_ok=True)
     model_name = args.pimcomp_model_name or f"compare_{model_path.stem}"
     frontend_json = frontend_json_dir / f"{model_name}.json"
+    print_step("Compile PIMCOMP", stage="Compile PIM")
     # The original PIMCOMP frontend rewrites its input ONNX while loading it.
     # Isolate that mutation without changing the model sent through PIMCOMP.
     with TemporaryDirectory(prefix="pimcomp-model-") as temp_dir:
@@ -586,6 +592,7 @@ def compile_pimcomp(
             timeout_sec=args.timeout_seconds,
             steps=steps,
             stage="Compile PIM",
+            print_header=False,
         )
     backend_cmd = [
         str(args.pimcomp_dir / "build" / "PIMCOMP-NN"),
@@ -603,6 +610,7 @@ def compile_pimcomp(
             timeout_sec=args.timeout_seconds,
             steps=steps,
             stage="Compile PIM",
+            print_header=False,
         )
     finally:
         shutil.rmtree(frontend_json_dir.parent, ignore_errors=True)
@@ -1014,6 +1022,28 @@ def optional_path(path: Path | None) -> str | None:
     return str(path) if path is not None else None
 
 
+def path_from_report(report: dict[str, Any], name: str) -> Path | None:
+    value = report.get("paths", {}).get(name)
+    return Path(value) if value else None
+
+
+def restore_side_records(
+    report: dict[str, Any],
+    side: str,
+    failures: list[dict[str, str]],
+    steps: list[StepRecord],
+) -> None:
+    marker = side.upper()
+    failures.extend(
+        failure for failure in report.get("failures", [])
+        if marker in failure.get("stage", "").upper()
+    )
+    steps.extend(
+        StepRecord(**step) for step in report.get("steps", [])
+        if marker in step.get("name", "").upper()
+    )
+
+
 def default_common_dir(out_dir: Path) -> Path:
     if out_dir.name == "latency":
         return out_dir.parents[1] / "common"
@@ -1280,6 +1310,11 @@ def main():
         type=Path,
         help="Reuse a directory containing PIMCOMP SimulationInfo.gz, VerificationInfo.json, and MappingResult.txt.",
     )
+    parser.add_argument(
+        "--reuse-pimcomp-report",
+        type=Path,
+        help="Preserve the PIMCOMP side of an existing comparison report without rerunning it.",
+    )
     parser.add_argument("--skip-pimsim-nn", action="store_true")
     parser.add_argument("--verbose-raptor-compile", action="store_true")
     parser.add_argument("--raptor-extra-arg", action="append", default=[])
@@ -1289,6 +1324,8 @@ def main():
         help="Return a non-zero status if a stage or semantic validation fails.",
     )
     args = parser.parse_args()
+    if args.reuse_pimcomp_dir is not None and args.reuse_pimcomp_report is not None:
+        parser.error("--reuse-pimcomp-dir and --reuse-pimcomp-report are mutually exclusive")
     if args.pimsim_time_ms <= 0:
         parser.error("--pimsim-time-ms must be positive")
     if args.timeout_seconds < 0:
@@ -1340,6 +1377,7 @@ def main():
     verification_info: Path | None = None
     simulation_info: Path | None = None
     pimcomp_export_dir: Path | None = None
+    pimcomp_pimsim_dir: Path | None = None
     pimcomp_model_path: Path | None = None
     pimsim_config: Path | None = None
     reuse_raptor = args.reuse_raptor_report is not None
@@ -1360,11 +1398,14 @@ def main():
         inputs_desc, outputs_desc, arrays_in_order = model_io
         runtime_inputs = arrays_in_order
 
-    reuse_pimcomp = args.reuse_pimcomp_dir is not None
+    reused_pimcomp_report_path = args.reuse_pimcomp_report
-    if reuse_pimcomp:
+    if args.reuse_pimcomp_dir is not None:
         reused_pimcomp_report_path = args.reuse_pimcomp_dir.resolve().parent / "comparison_report.json"
+    reuse_pimcomp = reused_pimcomp_report_path is not None
+    if reuse_pimcomp:
+        reused_pimcomp_report_path = reused_pimcomp_report_path.resolve()
         if not reused_pimcomp_report_path.exists():
-            raise ValueError(f"Missing PIMCOMP report beside {args.reuse_pimcomp_dir}")
+            raise ValueError(f"Missing PIMCOMP report: {reused_pimcomp_report_path}")
         with open(reused_pimcomp_report_path, "r", encoding="utf-8") as f:
             reused_pimcomp = json.load(f)
         if reused_pimcomp.get("pimcomp_model_source") != "original_onnx":
@@ -1372,9 +1413,13 @@ def main():
         pimcomp_validation = CompareResult(**reused_pimcomp["pimcomp_validation"])
         pimcomp_perf = reused_pimcomp["pimcomp_performance"]
         pimcomp_instr = reused_pimcomp["pimcomp_instruction_summary"]
-        reused_config = reused_pimcomp.get("paths", {}).get("pimsim_config")
+        simulation_info = path_from_report(reused_pimcomp, "pimcomp_simulation_info")
-        if reused_config:
+        pimcomp_export_dir = path_from_report(reused_pimcomp, "pimcomp_exported_pim")
-            pimsim_config = Path(reused_config)
+        pimcomp_pimsim_dir = path_from_report(reused_pimcomp, "pimcomp_pimsim_nn")
+        if pimcomp_pimsim_dir is None and (reused_pimcomp_report_path.parent / "pimsim_nn").is_dir():
+            pimcomp_pimsim_dir = reused_pimcomp_report_path.parent / "pimsim_nn"
+        pimsim_config = path_from_report(reused_pimcomp, "pimsim_config")
+        restore_side_records(reused_pimcomp, "pimcomp", failures, steps)
 
     if reuse_raptor and model_io is not None:
         reuse_report_path = args.reuse_raptor_report.resolve()
@@ -1384,8 +1429,9 @@ def main():
         if reused_hardware != hardware:
             raise ValueError(f"Reused Raptor hardware differs: {reused_hardware} != {hardware}")
         reference_dir = common_dir / "outputs"
-        reused_raptor_pim = reused["paths"].get("raptor_pim")
+        runner_path = path_from_report(reused, "reference_runner")
-        raptor_pim_dir = Path(reused_raptor_pim) if reused_raptor_pim else None
+        raptor_pim_dir = path_from_report(reused, "raptor_pim")
+        raptor_pimsim_dir = path_from_report(reused, "raptor_pimsim_nn")
         arrays_in_order = load_saved_inputs(
             inputs_desc,
             common_dir / "inputs",
@@ -1395,6 +1441,7 @@ def main():
         raptor_perf = reused["raptor_performance"]
         raptor_instr = reused["raptor_instruction_summary"]
         raptor_pass_timings = reused["raptor_pass_timings"]
+        restore_side_records(reused, "raptor", failures, steps)
         print_step("Reuse Raptor")
         print(f"  Report: {reuse_report_path}")
 
@@ -1528,7 +1575,7 @@ def main():
             simulation_info = out_dir / "pimcomp/output/SimulationInfo.gz"
             print_step("Reuse PIMCOMP")
             print(f"  Directory: {reused_pimcomp_dir}")
-    else:
+    elif not reuse_pimcomp:
         compiled_pimcomp = try_stage(
             failures,
             "Compile PIMCOMP",
@@ -1614,10 +1661,12 @@ def main():
     if args.skip_pimsim_nn:
         if not reuse_raptor:
             raptor_perf = skipped_perf("Skipped by --skip-pimsim-nn")
+        if not reuse_pimcomp:
             pimcomp_perf = skipped_perf("Skipped by --skip-pimsim-nn")
     elif pimsim_config is None:
         if not reuse_raptor:
             raptor_perf = skipped_perf("pimsim-nn config is not available")
+        if not reuse_pimcomp:
             pimcomp_perf = skipped_perf("pimsim-nn config is not available")
     else:
         if not reuse_raptor and raptor_pim_dir is not None:
@@ -1645,7 +1694,8 @@ def main():
         elif not reuse_raptor:
             raptor_perf = skipped_perf("Raptor PIM directory is not available")
 
-        if not reuse_pimcomp and simulation_info is not None:
+        if not reuse_pimcomp:
+            if simulation_info is not None:
                 pimcomp_pimsim_dir = try_stage(
                     failures,
                     "Export PIMCOMP for pimsim-nn",
@@ -1676,7 +1726,8 @@ def main():
     elif not reuse_raptor:
         raptor_instr = empty_instruction_summary("Raptor PIM directory is not available")
 
-    if not reuse_pimcomp and simulation_info is not None and simulation_info.exists():
+    if not reuse_pimcomp:
+        if simulation_info is not None and simulation_info.exists():
             parsed = try_stage(failures, "Parse PIMCOMP instructions", parse_pimcomp_instructions, simulation_info)
             pimcomp_instr = parsed if parsed is not None else empty_instruction_summary(error="Failed to parse PIMCOMP instructions")
         else:
@@ -1712,6 +1763,7 @@ def main():
         "pimcomp_config": str(args.pimcomp_config),
         "raptor_extra_args": args.raptor_extra_arg,
         "reused_raptor_report": optional_path(args.reuse_raptor_report.resolve()) if reuse_raptor else None,
+        "reused_pimcomp_report": optional_path(reused_pimcomp_report_path) if reuse_pimcomp else None,
         "failures": failures,
         "steps": [asdict(step) for step in steps],
         "raptor_validation": asdict(raptor_validation),
@@ -1730,6 +1782,7 @@ def main():
             "raptor_pimsim_nn": optional_path(raptor_pimsim_dir),
             "pimcomp_simulation_info": optional_path(simulation_info),
             "pimcomp_exported_pim": optional_path(pimcomp_export_dir),
+            "pimcomp_pimsim_nn": optional_path(pimcomp_pimsim_dir),
             "pimsim_config": optional_path(pimsim_config),
             "report_markdown": str(report_path),
         },
@@ -1739,15 +1792,28 @@ def main():
         json.dump(json_report, f, indent=2)
         f.write("\n")
 
-    failed_steps = any(step.status != "passed" for step in steps)
+    partial_side = "pimcomp" if reuse_raptor else "raptor" if args.reuse_pimcomp_report else None
+    other_side = "RAPTOR" if partial_side == "pimcomp" else "PIMCOMP"
+    relevant_failures = (
+        failures if partial_side is None
+        else [failure for failure in failures if other_side not in failure["stage"].upper()]
+    )
+    relevant_steps = (
+        steps if partial_side is None
+        else [step for step in steps if other_side not in step.name.upper()]
+    )
+    relevant_validations = (
+        (raptor_validation, pimcomp_validation)
+        if partial_side is None
+        else (raptor_validation,) if partial_side == "raptor" else (pimcomp_validation,)
+    )
+    failed_steps = any(step.status != "passed" for step in relevant_steps)
     functional_failure = any(
         result.status == "done" and not result.passed
-        for result in (raptor_validation, pimcomp_validation)
+        for result in relevant_validations
     )
-    failed = bool(failures or failed_steps or functional_failure)
+    failed = bool(relevant_failures or failed_steps or functional_failure)
-    result = "FAIL" if args.fail_on_error and failed else "DONE" if failed else "PASS"
+    print("\n" + Style.BRIGHT + Fore.GREEN + "[Completed]" + Style.RESET_ALL)
-    color = Fore.RED if result == "FAIL" else Fore.YELLOW if result == "DONE" else Fore.GREEN
-    print("\n" + Style.BRIGHT + f"Result: {color}{result}" + Style.RESET_ALL)
     print(f"  Report: {report_path}")
     print(f"  JSON:   {json_path}")
     if failures or failed_steps:

validation/tools/pim/pimcomp/compare/run_pimcomp_paper_latency.py

@@ -101,8 +101,8 @@ def write_results_csv(
 ) -> Path:
     output = (root or SUITE) / "results.csv"
     fields = (
-        "model",
         "arch",
+        "model",
         "mode",
         "raptor_pipeline",
         "pimcomp_pipeline",
@@ -185,12 +185,26 @@ def write_results_csv(
                     pimcomp_energy_pj=format_value(pimcomp_values["energy"]),
                 )
                 if raptor_metric is not None and pimcomp_metric is not None:
+                    min_metric = min(raptor_metric, pimcomp_metric)
                     row.update(
                         better_compiler=comparison_winner(comparison_mode, raptor_metric, pimcomp_metric),
-                        speedup=f"{max(raptor_metric, pimcomp_metric) / min(raptor_metric, pimcomp_metric):.2f}",
+                        speedup=(
+                            f"{max(raptor_metric, pimcomp_metric) / min_metric:.2f}"
+                            if min_metric > 0
+                            else "NA"
+                        ),
                     )
             rows.append(row)
-    rows.sort(key=lambda row: (list(MODELS).index(row["model"]), row["mode"], int(row["raptor_pipeline"])))
+    model_order = {name: index for index, name in enumerate(MODELS)}
+    mode_order = {"latency": 0, "throughput": 1}
+    rows.sort(
+        key=lambda row: (
+            row["arch"],
+            model_order[row["model"]],
+            mode_order[row["mode"]],
+            int(row["raptor_pipeline"]),
+        )
+    )
     with open(output, "w", newline="", encoding="utf-8") as f:
         writer = csv.DictWriter(f, fieldnames=fields, lineterminator="\n")
         writer.writeheader()
@@ -213,15 +227,20 @@ def performance_values(performance: dict) -> dict[str, float | None]:
     }
 
 
-def comparison_passed(report: dict) -> bool:
+def comparison_passed(report: dict, compiler: str | None = None) -> bool:
-    if report.get("failures"):
+    other_compiler = "PIMCOMP" if compiler == "raptor" else "RAPTOR"
+    if any(
+        compiler is None or other_compiler not in failure.get("stage", "").upper()
+        for failure in report.get("failures", [])
+    ):
         return False
-    for key in ("raptor_validation", "pimcomp_validation"):
+    compilers = (compiler,) if compiler is not None else ("raptor", "pimcomp")
-        result = report.get(key) or {}
+    for name in compilers:
+        result = report.get(f"{name}_validation") or {}
         if result.get("status") != "done" or not result.get("passed"):
             return False
-    for key in ("raptor_performance", "pimcomp_performance"):
+    for name in compilers:
-        performance = report.get(key) or {}
+        performance = report.get(f"{name}_performance") or {}
         if performance.get("error") or performance.get("skipped"):
             return False
     return True
@@ -249,6 +268,13 @@ def print_stage(title: str, color: str) -> None:
     print("\n" + Style.BRIGHT + color + f"[{title}]" + Style.RESET_ALL, flush=True)
 
 
+def print_completed(label: str, output: str = "") -> None:
+    print("\n" + Style.BRIGHT + Fore.CYAN + f"[Completed {label}]" + Style.RESET_ALL)
+    if output:
+        print(output, end="" if output.endswith("\n") else "\n")
+    print("=" * 72, flush=True)
+
+
 def run(command: list[str], *, dry_run: bool, check: bool = True) -> int:
     print(f"  cwd: {REPO}", flush=True)
     print(f"  $ {shlex.join(command)}", flush=True)
@@ -277,7 +303,7 @@ def run_comparison_job(job: tuple[str, list[str], Path | None]) -> tuple[str, in
         os.dup2(saved_stderr, 2)
         os.close(saved_stdout)
         os.close(saved_stderr)
-    return label, returncode, str(log_path)
+    return label, returncode, log_path.read_text(encoding="utf-8", errors="replace")
 
 
 def validate_pimcomp_source() -> None:
@@ -300,6 +326,7 @@ def comparison_command(
     timeout: float,
     reuse_raptor_report: Path | None = None,
     reuse_pimcomp_dir: Path | None = None,
+    reuse_pimcomp_report: Path | None = None,
 ) -> list[str]:
     time_args = ["--pimsim-time-ms", str(pimsim_time_ms)] if mode == "throughput" else []
     reuse_args = []
@@ -307,6 +334,8 @@ def comparison_command(
         reuse_args.extend(["--reuse-raptor-report", str(reuse_raptor_report)])
     if reuse_pimcomp_dir is not None:
         reuse_args.extend(["--reuse-pimcomp-dir", str(reuse_pimcomp_dir)])
+    if reuse_pimcomp_report is not None:
+        reuse_args.extend(["--reuse-pimcomp-report", str(reuse_pimcomp_report)])
     return [
         sys.executable,
         str(COMPARE),
@@ -357,11 +386,7 @@ def comparison_command_for(
     *,
     reuse_shared_pimcomp: bool,
 ) -> list[str]:
-    reuse_pimcomp_dir = None
+    report = spec.output_dir / "pimcomp/comparison_report.json"
-    if reuse_shared_pimcomp:
-        reuse_pimcomp_dir = spec.shared_pimcomp_dir
-    elif args.only == "raptor":
-        reuse_pimcomp_dir = existing_pimcomp_dir(spec.output_dir)
     return comparison_command(
         spec.model,
         spec.output_dir,
@@ -373,11 +398,12 @@ def comparison_command_for(
         args.pimsim_time_ms,
         args.timeout_seconds,
         reuse_raptor_report=(
-            spec.output_dir / "pimcomp/comparison_report.json"
+            report
             if args.only == "pimcomp"
             else None
         ),
-        reuse_pimcomp_dir=reuse_pimcomp_dir,
+        reuse_pimcomp_dir=spec.shared_pimcomp_dir if reuse_shared_pimcomp and args.only != "raptor" else None,
+        reuse_pimcomp_report=report if args.only == "raptor" else None,
     )
 
 
@@ -388,11 +414,6 @@ def pimcomp_artifact_ready(output_dir: Path) -> bool:
     ) and (output_dir.parent / "comparison_report.json").is_file()
 
 
-def existing_pimcomp_dir(comparison_dir: Path) -> Path | None:
-    output_dir = comparison_dir / "pimcomp/output"
-    return output_dir if pimcomp_artifact_ready(output_dir) else None
-
-
 def run_comparison_jobs(
     comparison_jobs: list[tuple[str, list[str], Path | None]],
     max_workers: int,
@@ -420,11 +441,8 @@ def run_comparison_jobs(
             )
         )
         failed = []
-        for label, returncode, result_log in completed:
+        for label, returncode, output in completed:
-            if result_log:
+            print_completed(label, output or "")
-                output = Path(result_log).read_text(encoding="utf-8", errors="replace")
-                if output:
-                    print(output, end="" if output.endswith("\n") else "\n")
             if returncode:
                 failed.append(label)
     return failed, log_offset + len(jobs)
@@ -456,13 +474,6 @@ def config_path(arch: str, mode: str, sim_time_ms: int, *, write: bool) -> Path:
     return path
 
 
-def core_count(config: Path) -> int:
-    if not config.exists() and config.name.startswith("throughput_config_"):
-        config = config.with_name("throughput_config_1000ms.json")
-    with open(config, encoding="utf-8") as f:
-        return int(json.load(f)["chip_config"]["core_cnt"])
-
-
 def main() -> int:
     parser = argparse.ArgumentParser(
         description="Compare supported PIMCOMP models with Raptor latency and throughput schedules."
@@ -490,7 +501,7 @@ def main() -> int:
     parser.add_argument(
         "--only",
         choices=("raptor", "pimcomp"),
-        help="Re-run only this compiler and reuse the other side's existing artifacts.",
+        help="Re-run only this compiler's compile, validation, and simulation stages; preserve the other side from its report.",
     )
     parser.add_argument(
         "--pipeline",
@@ -552,22 +563,6 @@ def main() -> int:
             mode: config_path(arch, mode, args.pimsim_time_ms, write=not args.dry_run)
             for mode, _, _ in comparisons
         }
-        unsupported = [
-            pipeline for mode, pipeline, _ in comparisons
-            if mode == "throughput" and core_count(configs[mode]) % pipeline
-        ]
-        if unsupported:
-            if args.pipeline is not None:
-                parser.error(
-                    f"{arch} has {core_count(configs['throughput'])} cores; "
-                    f"throughput pipelines must divide that count (invalid: {unsupported})"
-                )
-            print(
-                Fore.YELLOW
-                + f"Skipping unsupported {arch} throughput pipeline(s): {unsupported}"
-                + Style.RESET_ALL
-            )
-            comparisons = tuple(comparison for comparison in comparisons if comparison[1] not in unsupported)
         comparisons_by_arch[arch] = comparisons
         configs_by_arch[arch] = configs
 
@@ -581,19 +576,14 @@ def main() -> int:
             for name in args.models:
                 for mode, pipeline, _ in comparisons_by_arch[arch]:
                     comparison_dir = result_dir(out_dir, name, arch, mode, pipeline)
-                    required = (
+                    required = comparison_dir / "pimcomp/comparison_report.json"
-                        comparison_dir / "pimcomp/comparison_report.json"
+                    if not required.exists():
-                        if args.only == "pimcomp"
+                        missing_reuse.append(str(required))
-                        else existing_pimcomp_dir(comparison_dir)
-                    )
-                    if required is None or not required.exists():
-                        missing_reuse.append(str(required or comparison_dir / "pimcomp"))
                     elif args.only == "raptor":
-                        report = comparison_dir / "pimcomp/comparison_report.json"
+                        if json.loads(required.read_text(encoding="utf-8")).get(
-                        if not report.exists() or json.loads(report.read_text(encoding="utf-8")).get(
                             "pimcomp_model_source"
                         ) != "original_onnx":
-                            missing_reuse.append(f"{report} (not generated from the original ONNX model)")
+                            missing_reuse.append(f"{required} (not generated from the original ONNX model)")
         if missing_reuse:
             parser.error(
                 f"--only {args.only} needs existing comparison artifacts: "
@@ -609,11 +599,16 @@ def main() -> int:
     print(f"Modes: {', '.join(args.mode)}")
     print(f"Throughput pimsim time: {args.pimsim_time_ms} ms")
     print(f"Max parallel jobs: {args.jobs}")
+    print(
+        f"Comparison jobs: "
+        f"{sum(len(args.models) * len(comparisons) for comparisons in comparisons_by_arch.values())}"
+    )
     print(f"Results root: {out_dir or SUITE}")
     print("=" * 72)
 
     print_stage("Prepare shared artifacts", STAGE_COLORS["Build Runner"])
     for name in args.models:
+        try:
             run(
                 prepare_common_command(
                     MODELS[name],
@@ -622,6 +617,8 @@ def main() -> int:
                 ),
                 dry_run=args.dry_run,
             )
+        finally:
+            print_completed(f"shared artifacts: {name}")
 
     failed = []
     comparison_specs: list[ComparisonSpec] = []
@@ -629,7 +626,7 @@ def main() -> int:
     for arch in args.arch:
         comparisons = comparisons_by_arch[arch]
         configs = configs_by_arch[arch]
-        for index, name in enumerate(args.models, start=1):
+        for name in args.models:
             for mode, pipeline, pimcomp_pipeline in comparisons:
                 model_result_dir = result_dir(out_dir, name, arch, mode, pipeline)
                 label = f"{arch}/{name}/{mode}/pipeline{pipeline}"
@@ -639,11 +636,6 @@ def main() -> int:
                 if anchor:
                     shared_pimcomp_dir = model_result_dir / "pimcomp/output"
                     shared_pimcomp_by_group[group] = shared_pimcomp_dir
-                print(
-                    "\n" + Fore.CYAN + f"[{arch} {index}/{len(args.models)}]" + Style.RESET_ALL
-                    + f" {Style.BRIGHT}Comparing {name} ({mode}, pipeline={pipeline}){Style.RESET_ALL}",
-                    flush=True,
-                )
                 comparison_specs.append(
                     ComparisonSpec(
                         label=label,
@@ -669,17 +661,20 @@ def main() -> int:
             if run(command, dry_run=True, check=False):
                 failed.append(spec.label)
     elif comparison_specs:
-        print(f"Running {len(comparison_specs)} comparison(s)")
+        anchor_specs = comparison_specs if args.only == "raptor" else [
+            spec for spec in comparison_specs if spec.anchor
+        ]
         anchor_jobs = [
             (
                 spec.label,
                 comparison_command_for(spec, args, reuse_shared_pimcomp=False),
                 None,
             )
-            for spec in comparison_specs
+            for spec in anchor_specs
-            if spec.anchor
+        ]
+        dependent_specs = [] if args.only == "raptor" else [
+            spec for spec in comparison_specs if not spec.anchor
         ]
-        dependent_specs = [spec for spec in comparison_specs if not spec.anchor]
         print_directly = min(args.jobs, len(comparison_specs)) == 1
         with (nullcontext(None) if print_directly else TemporaryDirectory(prefix="raptor-pimcomp-")) as log_dir:
             anchor_failed, log_offset = run_comparison_jobs(
@@ -729,7 +724,8 @@ def main() -> int:
                 label = f"{arch}/{name}/{mode}/pipeline{pipeline}"
                 report_path = result_dir(out_dir, name, arch, mode, pipeline) / "pimcomp/comparison_report.json"
                 if not report_path.exists() or not comparison_passed(
-                    json.loads(report_path.read_text(encoding="utf-8"))
+                    json.loads(report_path.read_text(encoding="utf-8")),
+                    args.only,
                 ):
                     if label not in failed:
                         failed.append(label)