diff --git a/validation/tools/pim/pimcomp/compare/compare_raptor_pimcomp.py b/validation/tools/pim/pimcomp/compare/compare_raptor_pimcomp.py index 1de5637..d6c128b 100755 --- a/validation/tools/pim/pimcomp/compare/compare_raptor_pimcomp.py +++ b/validation/tools/pim/pimcomp/compare/compare_raptor_pimcomp.py @@ -145,8 +145,13 @@ def run_logged( timeout_sec: float, steps: list[StepRecord], stage: str | None = None, + print_header: bool = True, ) -> str: - print_step(name, cmd, cwd, stage) + if print_header: + print_step(name, cmd, cwd, stage) + else: + print(f" cwd: {cwd or REPO}") + print(f" $ {shell_join(cmd)}") start = time.perf_counter() command = shell_join(cmd) try: @@ -446,7 +451,7 @@ def compile_raptor_target( "--pim-emit-json", *args.raptor_extra_arg, ] - print_step("Compile Raptor PIM", cmd, REPO, "Compile PIM") + print_step("Compile Raptor", cmd, REPO, "Compile PIM") start = time.perf_counter() command = shell_join(cmd) raptor_extra_args = [ @@ -566,6 +571,7 @@ def compile_pimcomp( (pimcomp_output_dir / name).unlink(missing_ok=True) model_name = args.pimcomp_model_name or f"compare_{model_path.stem}" frontend_json = frontend_json_dir / f"{model_name}.json" + print_step("Compile PIMCOMP", stage="Compile PIM") # The original PIMCOMP frontend rewrites its input ONNX while loading it. # Isolate that mutation without changing the model sent through PIMCOMP. with TemporaryDirectory(prefix="pimcomp-model-") as temp_dir: @@ -586,6 +592,7 @@ def compile_pimcomp( timeout_sec=args.timeout_seconds, steps=steps, stage="Compile PIM", + print_header=False, ) backend_cmd = [ str(args.pimcomp_dir / "build" / "PIMCOMP-NN"), @@ -603,6 +610,7 @@ def compile_pimcomp( timeout_sec=args.timeout_seconds, steps=steps, stage="Compile PIM", + print_header=False, ) finally: shutil.rmtree(frontend_json_dir.parent, ignore_errors=True) @@ -1014,6 +1022,28 @@ def optional_path(path: Path | None) -> str | None: return str(path) if path is not None else None +def path_from_report(report: dict[str, Any], name: str) -> Path | None: + value = report.get("paths", {}).get(name) + return Path(value) if value else None + + +def restore_side_records( + report: dict[str, Any], + side: str, + failures: list[dict[str, str]], + steps: list[StepRecord], +) -> None: + marker = side.upper() + failures.extend( + failure for failure in report.get("failures", []) + if marker in failure.get("stage", "").upper() + ) + steps.extend( + StepRecord(**step) for step in report.get("steps", []) + if marker in step.get("name", "").upper() + ) + + def default_common_dir(out_dir: Path) -> Path: if out_dir.name == "latency": return out_dir.parents[1] / "common" @@ -1280,6 +1310,11 @@ def main(): type=Path, help="Reuse a directory containing PIMCOMP SimulationInfo.gz, VerificationInfo.json, and MappingResult.txt.", ) + parser.add_argument( + "--reuse-pimcomp-report", + type=Path, + help="Preserve the PIMCOMP side of an existing comparison report without rerunning it.", + ) parser.add_argument("--skip-pimsim-nn", action="store_true") parser.add_argument("--verbose-raptor-compile", action="store_true") parser.add_argument("--raptor-extra-arg", action="append", default=[]) @@ -1289,6 +1324,8 @@ def main(): help="Return a non-zero status if a stage or semantic validation fails.", ) args = parser.parse_args() + if args.reuse_pimcomp_dir is not None and args.reuse_pimcomp_report is not None: + parser.error("--reuse-pimcomp-dir and --reuse-pimcomp-report are mutually exclusive") if args.pimsim_time_ms <= 0: parser.error("--pimsim-time-ms must be positive") if args.timeout_seconds < 0: @@ -1340,6 +1377,7 @@ def main(): verification_info: Path | None = None simulation_info: Path | None = None pimcomp_export_dir: Path | None = None + pimcomp_pimsim_dir: Path | None = None pimcomp_model_path: Path | None = None pimsim_config: Path | None = None reuse_raptor = args.reuse_raptor_report is not None @@ -1360,11 +1398,14 @@ def main(): inputs_desc, outputs_desc, arrays_in_order = model_io runtime_inputs = arrays_in_order - reuse_pimcomp = args.reuse_pimcomp_dir is not None - if reuse_pimcomp: + reused_pimcomp_report_path = args.reuse_pimcomp_report + if args.reuse_pimcomp_dir is not None: reused_pimcomp_report_path = args.reuse_pimcomp_dir.resolve().parent / "comparison_report.json" + reuse_pimcomp = reused_pimcomp_report_path is not None + if reuse_pimcomp: + reused_pimcomp_report_path = reused_pimcomp_report_path.resolve() if not reused_pimcomp_report_path.exists(): - raise ValueError(f"Missing PIMCOMP report beside {args.reuse_pimcomp_dir}") + raise ValueError(f"Missing PIMCOMP report: {reused_pimcomp_report_path}") with open(reused_pimcomp_report_path, "r", encoding="utf-8") as f: reused_pimcomp = json.load(f) if reused_pimcomp.get("pimcomp_model_source") != "original_onnx": @@ -1372,9 +1413,13 @@ def main(): pimcomp_validation = CompareResult(**reused_pimcomp["pimcomp_validation"]) pimcomp_perf = reused_pimcomp["pimcomp_performance"] pimcomp_instr = reused_pimcomp["pimcomp_instruction_summary"] - reused_config = reused_pimcomp.get("paths", {}).get("pimsim_config") - if reused_config: - pimsim_config = Path(reused_config) + simulation_info = path_from_report(reused_pimcomp, "pimcomp_simulation_info") + pimcomp_export_dir = path_from_report(reused_pimcomp, "pimcomp_exported_pim") + pimcomp_pimsim_dir = path_from_report(reused_pimcomp, "pimcomp_pimsim_nn") + if pimcomp_pimsim_dir is None and (reused_pimcomp_report_path.parent / "pimsim_nn").is_dir(): + pimcomp_pimsim_dir = reused_pimcomp_report_path.parent / "pimsim_nn" + pimsim_config = path_from_report(reused_pimcomp, "pimsim_config") + restore_side_records(reused_pimcomp, "pimcomp", failures, steps) if reuse_raptor and model_io is not None: reuse_report_path = args.reuse_raptor_report.resolve() @@ -1384,8 +1429,9 @@ def main(): if reused_hardware != hardware: raise ValueError(f"Reused Raptor hardware differs: {reused_hardware} != {hardware}") reference_dir = common_dir / "outputs" - reused_raptor_pim = reused["paths"].get("raptor_pim") - raptor_pim_dir = Path(reused_raptor_pim) if reused_raptor_pim else None + runner_path = path_from_report(reused, "reference_runner") + raptor_pim_dir = path_from_report(reused, "raptor_pim") + raptor_pimsim_dir = path_from_report(reused, "raptor_pimsim_nn") arrays_in_order = load_saved_inputs( inputs_desc, common_dir / "inputs", @@ -1395,6 +1441,7 @@ def main(): raptor_perf = reused["raptor_performance"] raptor_instr = reused["raptor_instruction_summary"] raptor_pass_timings = reused["raptor_pass_timings"] + restore_side_records(reused, "raptor", failures, steps) print_step("Reuse Raptor") print(f" Report: {reuse_report_path}") @@ -1528,7 +1575,7 @@ def main(): simulation_info = out_dir / "pimcomp/output/SimulationInfo.gz" print_step("Reuse PIMCOMP") print(f" Directory: {reused_pimcomp_dir}") - else: + elif not reuse_pimcomp: compiled_pimcomp = try_stage( failures, "Compile PIMCOMP", @@ -1614,11 +1661,13 @@ def main(): if args.skip_pimsim_nn: if not reuse_raptor: raptor_perf = skipped_perf("Skipped by --skip-pimsim-nn") - pimcomp_perf = skipped_perf("Skipped by --skip-pimsim-nn") + if not reuse_pimcomp: + pimcomp_perf = skipped_perf("Skipped by --skip-pimsim-nn") elif pimsim_config is None: if not reuse_raptor: raptor_perf = skipped_perf("pimsim-nn config is not available") - pimcomp_perf = skipped_perf("pimsim-nn config is not available") + if not reuse_pimcomp: + pimcomp_perf = skipped_perf("pimsim-nn config is not available") else: if not reuse_raptor and raptor_pim_dir is not None: raptor_pimsim_dir = try_stage( @@ -1645,30 +1694,31 @@ def main(): elif not reuse_raptor: raptor_perf = skipped_perf("Raptor PIM directory is not available") - if not reuse_pimcomp and simulation_info is not None: - pimcomp_pimsim_dir = try_stage( - failures, - "Export PIMCOMP for pimsim-nn", - export_pimcomp_for_pimsim_nn, - simulation_info, - out_dir / "pimcomp/pimsim_nn", - ) - if pimcomp_pimsim_dir is not None: - perf = try_stage( + if not reuse_pimcomp: + if simulation_info is not None: + pimcomp_pimsim_dir = try_stage( failures, - "Non-Functional Simulation PIMCOMP", - run_pimsim_nn, - "Non-Functional Simulation PIMCOMP", - pimcomp_pimsim_dir, - pimsim_config, - steps, - args, + "Export PIMCOMP for pimsim-nn", + export_pimcomp_for_pimsim_nn, + simulation_info, + out_dir / "pimcomp/pimsim_nn", ) - pimcomp_perf = perf if perf is not None else failed_perf("pimsim-nn PIMCOMP failed") + if pimcomp_pimsim_dir is not None: + perf = try_stage( + failures, + "Non-Functional Simulation PIMCOMP", + run_pimsim_nn, + "Non-Functional Simulation PIMCOMP", + pimcomp_pimsim_dir, + pimsim_config, + steps, + args, + ) + pimcomp_perf = perf if perf is not None else failed_perf("pimsim-nn PIMCOMP failed") + else: + pimcomp_perf = failed_perf("PIMCOMP pimsim-nn export failed") else: - pimcomp_perf = failed_perf("PIMCOMP pimsim-nn export failed") - else: - pimcomp_perf = skipped_perf("PIMCOMP SimulationInfo.gz is not available") + pimcomp_perf = skipped_perf("PIMCOMP SimulationInfo.gz is not available") if not reuse_raptor and raptor_pim_dir is not None and raptor_pim_dir.exists(): parsed = try_stage(failures, "Parse Raptor instructions", parse_raptor_instructions, raptor_pim_dir) @@ -1676,11 +1726,12 @@ def main(): elif not reuse_raptor: raptor_instr = empty_instruction_summary("Raptor PIM directory is not available") - if not reuse_pimcomp and simulation_info is not None and simulation_info.exists(): - parsed = try_stage(failures, "Parse PIMCOMP instructions", parse_pimcomp_instructions, simulation_info) - pimcomp_instr = parsed if parsed is not None else empty_instruction_summary(error="Failed to parse PIMCOMP instructions") - else: - pimcomp_instr = empty_instruction_summary("PIMCOMP SimulationInfo.gz is not available") + if not reuse_pimcomp: + if simulation_info is not None and simulation_info.exists(): + parsed = try_stage(failures, "Parse PIMCOMP instructions", parse_pimcomp_instructions, simulation_info) + pimcomp_instr = parsed if parsed is not None else empty_instruction_summary(error="Failed to parse PIMCOMP instructions") + else: + pimcomp_instr = empty_instruction_summary("PIMCOMP SimulationInfo.gz is not available") report_path = out_dir / "pimcomp/comparison_report.md" write_report( @@ -1712,6 +1763,7 @@ def main(): "pimcomp_config": str(args.pimcomp_config), "raptor_extra_args": args.raptor_extra_arg, "reused_raptor_report": optional_path(args.reuse_raptor_report.resolve()) if reuse_raptor else None, + "reused_pimcomp_report": optional_path(reused_pimcomp_report_path) if reuse_pimcomp else None, "failures": failures, "steps": [asdict(step) for step in steps], "raptor_validation": asdict(raptor_validation), @@ -1730,6 +1782,7 @@ def main(): "raptor_pimsim_nn": optional_path(raptor_pimsim_dir), "pimcomp_simulation_info": optional_path(simulation_info), "pimcomp_exported_pim": optional_path(pimcomp_export_dir), + "pimcomp_pimsim_nn": optional_path(pimcomp_pimsim_dir), "pimsim_config": optional_path(pimsim_config), "report_markdown": str(report_path), }, @@ -1739,15 +1792,28 @@ def main(): json.dump(json_report, f, indent=2) f.write("\n") - failed_steps = any(step.status != "passed" for step in steps) + partial_side = "pimcomp" if reuse_raptor else "raptor" if args.reuse_pimcomp_report else None + other_side = "RAPTOR" if partial_side == "pimcomp" else "PIMCOMP" + relevant_failures = ( + failures if partial_side is None + else [failure for failure in failures if other_side not in failure["stage"].upper()] + ) + relevant_steps = ( + steps if partial_side is None + else [step for step in steps if other_side not in step.name.upper()] + ) + relevant_validations = ( + (raptor_validation, pimcomp_validation) + if partial_side is None + else (raptor_validation,) if partial_side == "raptor" else (pimcomp_validation,) + ) + failed_steps = any(step.status != "passed" for step in relevant_steps) functional_failure = any( result.status == "done" and not result.passed - for result in (raptor_validation, pimcomp_validation) + for result in relevant_validations ) - failed = bool(failures or failed_steps or functional_failure) - result = "FAIL" if args.fail_on_error and failed else "DONE" if failed else "PASS" - color = Fore.RED if result == "FAIL" else Fore.YELLOW if result == "DONE" else Fore.GREEN - print("\n" + Style.BRIGHT + f"Result: {color}{result}" + Style.RESET_ALL) + failed = bool(relevant_failures or failed_steps or functional_failure) + print("\n" + Style.BRIGHT + Fore.GREEN + "[Completed]" + Style.RESET_ALL) print(f" Report: {report_path}") print(f" JSON: {json_path}") if failures or failed_steps: diff --git a/validation/tools/pim/pimcomp/compare/run_pimcomp_paper_latency.py b/validation/tools/pim/pimcomp/compare/run_pimcomp_paper_latency.py index fb1a6bc..a1e30a0 100755 --- a/validation/tools/pim/pimcomp/compare/run_pimcomp_paper_latency.py +++ b/validation/tools/pim/pimcomp/compare/run_pimcomp_paper_latency.py @@ -101,8 +101,8 @@ def write_results_csv( ) -> Path: output = (root or SUITE) / "results.csv" fields = ( - "model", "arch", + "model", "mode", "raptor_pipeline", "pimcomp_pipeline", @@ -185,12 +185,26 @@ def write_results_csv( pimcomp_energy_pj=format_value(pimcomp_values["energy"]), ) if raptor_metric is not None and pimcomp_metric is not None: + min_metric = min(raptor_metric, pimcomp_metric) row.update( better_compiler=comparison_winner(comparison_mode, raptor_metric, pimcomp_metric), - speedup=f"{max(raptor_metric, pimcomp_metric) / min(raptor_metric, pimcomp_metric):.2f}", + speedup=( + f"{max(raptor_metric, pimcomp_metric) / min_metric:.2f}" + if min_metric > 0 + else "NA" + ), ) rows.append(row) - rows.sort(key=lambda row: (list(MODELS).index(row["model"]), row["mode"], int(row["raptor_pipeline"]))) + model_order = {name: index for index, name in enumerate(MODELS)} + mode_order = {"latency": 0, "throughput": 1} + rows.sort( + key=lambda row: ( + row["arch"], + model_order[row["model"]], + mode_order[row["mode"]], + int(row["raptor_pipeline"]), + ) + ) with open(output, "w", newline="", encoding="utf-8") as f: writer = csv.DictWriter(f, fieldnames=fields, lineterminator="\n") writer.writeheader() @@ -213,15 +227,20 @@ def performance_values(performance: dict) -> dict[str, float | None]: } -def comparison_passed(report: dict) -> bool: - if report.get("failures"): +def comparison_passed(report: dict, compiler: str | None = None) -> bool: + other_compiler = "PIMCOMP" if compiler == "raptor" else "RAPTOR" + if any( + compiler is None or other_compiler not in failure.get("stage", "").upper() + for failure in report.get("failures", []) + ): return False - for key in ("raptor_validation", "pimcomp_validation"): - result = report.get(key) or {} + compilers = (compiler,) if compiler is not None else ("raptor", "pimcomp") + for name in compilers: + result = report.get(f"{name}_validation") or {} if result.get("status") != "done" or not result.get("passed"): return False - for key in ("raptor_performance", "pimcomp_performance"): - performance = report.get(key) or {} + for name in compilers: + performance = report.get(f"{name}_performance") or {} if performance.get("error") or performance.get("skipped"): return False return True @@ -249,6 +268,13 @@ def print_stage(title: str, color: str) -> None: print("\n" + Style.BRIGHT + color + f"[{title}]" + Style.RESET_ALL, flush=True) +def print_completed(label: str, output: str = "") -> None: + print("\n" + Style.BRIGHT + Fore.CYAN + f"[Completed {label}]" + Style.RESET_ALL) + if output: + print(output, end="" if output.endswith("\n") else "\n") + print("=" * 72, flush=True) + + def run(command: list[str], *, dry_run: bool, check: bool = True) -> int: print(f" cwd: {REPO}", flush=True) print(f" $ {shlex.join(command)}", flush=True) @@ -277,7 +303,7 @@ def run_comparison_job(job: tuple[str, list[str], Path | None]) -> tuple[str, in os.dup2(saved_stderr, 2) os.close(saved_stdout) os.close(saved_stderr) - return label, returncode, str(log_path) + return label, returncode, log_path.read_text(encoding="utf-8", errors="replace") def validate_pimcomp_source() -> None: @@ -300,6 +326,7 @@ def comparison_command( timeout: float, reuse_raptor_report: Path | None = None, reuse_pimcomp_dir: Path | None = None, + reuse_pimcomp_report: Path | None = None, ) -> list[str]: time_args = ["--pimsim-time-ms", str(pimsim_time_ms)] if mode == "throughput" else [] reuse_args = [] @@ -307,6 +334,8 @@ def comparison_command( reuse_args.extend(["--reuse-raptor-report", str(reuse_raptor_report)]) if reuse_pimcomp_dir is not None: reuse_args.extend(["--reuse-pimcomp-dir", str(reuse_pimcomp_dir)]) + if reuse_pimcomp_report is not None: + reuse_args.extend(["--reuse-pimcomp-report", str(reuse_pimcomp_report)]) return [ sys.executable, str(COMPARE), @@ -357,11 +386,7 @@ def comparison_command_for( *, reuse_shared_pimcomp: bool, ) -> list[str]: - reuse_pimcomp_dir = None - if reuse_shared_pimcomp: - reuse_pimcomp_dir = spec.shared_pimcomp_dir - elif args.only == "raptor": - reuse_pimcomp_dir = existing_pimcomp_dir(spec.output_dir) + report = spec.output_dir / "pimcomp/comparison_report.json" return comparison_command( spec.model, spec.output_dir, @@ -373,11 +398,12 @@ def comparison_command_for( args.pimsim_time_ms, args.timeout_seconds, reuse_raptor_report=( - spec.output_dir / "pimcomp/comparison_report.json" + report if args.only == "pimcomp" else None ), - reuse_pimcomp_dir=reuse_pimcomp_dir, + reuse_pimcomp_dir=spec.shared_pimcomp_dir if reuse_shared_pimcomp and args.only != "raptor" else None, + reuse_pimcomp_report=report if args.only == "raptor" else None, ) @@ -388,11 +414,6 @@ def pimcomp_artifact_ready(output_dir: Path) -> bool: ) and (output_dir.parent / "comparison_report.json").is_file() -def existing_pimcomp_dir(comparison_dir: Path) -> Path | None: - output_dir = comparison_dir / "pimcomp/output" - return output_dir if pimcomp_artifact_ready(output_dir) else None - - def run_comparison_jobs( comparison_jobs: list[tuple[str, list[str], Path | None]], max_workers: int, @@ -420,11 +441,8 @@ def run_comparison_jobs( ) ) failed = [] - for label, returncode, result_log in completed: - if result_log: - output = Path(result_log).read_text(encoding="utf-8", errors="replace") - if output: - print(output, end="" if output.endswith("\n") else "\n") + for label, returncode, output in completed: + print_completed(label, output or "") if returncode: failed.append(label) return failed, log_offset + len(jobs) @@ -456,13 +474,6 @@ def config_path(arch: str, mode: str, sim_time_ms: int, *, write: bool) -> Path: return path -def core_count(config: Path) -> int: - if not config.exists() and config.name.startswith("throughput_config_"): - config = config.with_name("throughput_config_1000ms.json") - with open(config, encoding="utf-8") as f: - return int(json.load(f)["chip_config"]["core_cnt"]) - - def main() -> int: parser = argparse.ArgumentParser( description="Compare supported PIMCOMP models with Raptor latency and throughput schedules." @@ -490,7 +501,7 @@ def main() -> int: parser.add_argument( "--only", choices=("raptor", "pimcomp"), - help="Re-run only this compiler and reuse the other side's existing artifacts.", + help="Re-run only this compiler's compile, validation, and simulation stages; preserve the other side from its report.", ) parser.add_argument( "--pipeline", @@ -552,22 +563,6 @@ def main() -> int: mode: config_path(arch, mode, args.pimsim_time_ms, write=not args.dry_run) for mode, _, _ in comparisons } - unsupported = [ - pipeline for mode, pipeline, _ in comparisons - if mode == "throughput" and core_count(configs[mode]) % pipeline - ] - if unsupported: - if args.pipeline is not None: - parser.error( - f"{arch} has {core_count(configs['throughput'])} cores; " - f"throughput pipelines must divide that count (invalid: {unsupported})" - ) - print( - Fore.YELLOW - + f"Skipping unsupported {arch} throughput pipeline(s): {unsupported}" - + Style.RESET_ALL - ) - comparisons = tuple(comparison for comparison in comparisons if comparison[1] not in unsupported) comparisons_by_arch[arch] = comparisons configs_by_arch[arch] = configs @@ -581,19 +576,14 @@ def main() -> int: for name in args.models: for mode, pipeline, _ in comparisons_by_arch[arch]: comparison_dir = result_dir(out_dir, name, arch, mode, pipeline) - required = ( - comparison_dir / "pimcomp/comparison_report.json" - if args.only == "pimcomp" - else existing_pimcomp_dir(comparison_dir) - ) - if required is None or not required.exists(): - missing_reuse.append(str(required or comparison_dir / "pimcomp")) + required = comparison_dir / "pimcomp/comparison_report.json" + if not required.exists(): + missing_reuse.append(str(required)) elif args.only == "raptor": - report = comparison_dir / "pimcomp/comparison_report.json" - if not report.exists() or json.loads(report.read_text(encoding="utf-8")).get( + if json.loads(required.read_text(encoding="utf-8")).get( "pimcomp_model_source" ) != "original_onnx": - missing_reuse.append(f"{report} (not generated from the original ONNX model)") + missing_reuse.append(f"{required} (not generated from the original ONNX model)") if missing_reuse: parser.error( f"--only {args.only} needs existing comparison artifacts: " @@ -609,19 +599,26 @@ def main() -> int: print(f"Modes: {', '.join(args.mode)}") print(f"Throughput pimsim time: {args.pimsim_time_ms} ms") print(f"Max parallel jobs: {args.jobs}") + print( + f"Comparison jobs: " + f"{sum(len(args.models) * len(comparisons) for comparisons in comparisons_by_arch.values())}" + ) print(f"Results root: {out_dir or SUITE}") print("=" * 72) print_stage("Prepare shared artifacts", STAGE_COLORS["Build Runner"]) for name in args.models: - run( - prepare_common_command( - MODELS[name], - model_dir(out_dir, name) / "common", - args.timeout_seconds, - ), - dry_run=args.dry_run, - ) + try: + run( + prepare_common_command( + MODELS[name], + model_dir(out_dir, name) / "common", + args.timeout_seconds, + ), + dry_run=args.dry_run, + ) + finally: + print_completed(f"shared artifacts: {name}") failed = [] comparison_specs: list[ComparisonSpec] = [] @@ -629,7 +626,7 @@ def main() -> int: for arch in args.arch: comparisons = comparisons_by_arch[arch] configs = configs_by_arch[arch] - for index, name in enumerate(args.models, start=1): + for name in args.models: for mode, pipeline, pimcomp_pipeline in comparisons: model_result_dir = result_dir(out_dir, name, arch, mode, pipeline) label = f"{arch}/{name}/{mode}/pipeline{pipeline}" @@ -639,11 +636,6 @@ def main() -> int: if anchor: shared_pimcomp_dir = model_result_dir / "pimcomp/output" shared_pimcomp_by_group[group] = shared_pimcomp_dir - print( - "\n" + Fore.CYAN + f"[{arch} {index}/{len(args.models)}]" + Style.RESET_ALL - + f" {Style.BRIGHT}Comparing {name} ({mode}, pipeline={pipeline}){Style.RESET_ALL}", - flush=True, - ) comparison_specs.append( ComparisonSpec( label=label, @@ -669,17 +661,20 @@ def main() -> int: if run(command, dry_run=True, check=False): failed.append(spec.label) elif comparison_specs: - print(f"Running {len(comparison_specs)} comparison(s)") + anchor_specs = comparison_specs if args.only == "raptor" else [ + spec for spec in comparison_specs if spec.anchor + ] anchor_jobs = [ ( spec.label, comparison_command_for(spec, args, reuse_shared_pimcomp=False), None, ) - for spec in comparison_specs - if spec.anchor + for spec in anchor_specs + ] + dependent_specs = [] if args.only == "raptor" else [ + spec for spec in comparison_specs if not spec.anchor ] - dependent_specs = [spec for spec in comparison_specs if not spec.anchor] print_directly = min(args.jobs, len(comparison_specs)) == 1 with (nullcontext(None) if print_directly else TemporaryDirectory(prefix="raptor-pimcomp-")) as log_dir: anchor_failed, log_offset = run_comparison_jobs( @@ -729,7 +724,8 @@ def main() -> int: label = f"{arch}/{name}/{mode}/pipeline{pipeline}" report_path = result_dir(out_dir, name, arch, mode, pipeline) / "pimcomp/comparison_report.json" if not report_path.exists() or not comparison_passed( - json.loads(report_path.read_text(encoding="utf-8")) + json.loads(report_path.read_text(encoding="utf-8")), + args.only, ): if label not in failed: failed.append(label)