finally fast googlenet with correct latency artifacts for fair comparison

validation/tools/pimcomp/compare_raptor_pimcomp.py

@@ -24,9 +24,10 @@ import onnx
 from colorama import Fore, Style
 
 
-REPO = Path(__file__).resolve().parents[2]
+REPO = Path(__file__).resolve().parents[3]
 VALIDATION_DIR = REPO / "validation"
 PIMSIM_CONFIG_DIR = VALIDATION_DIR / "pimsim_configs/pimcomp"
+PIMCOMP_OUTPUT_FILES = ("SimulationInfo.gz", "VerificationInfo.json", "MappingResult.txt")
 sys.path.insert(0, str(VALIDATION_DIR))
 
 from raptor_validation.gen_network_runner import gen_network_runner  # noqa: E402
@@ -38,6 +39,10 @@ from raptor_validation.onnx_utils import (  # noqa: E402
     write_inputs_to_memory_bin,
 )
 from raptor_validation.raptor import compile_with_raptor  # noqa: E402
+from raptor_validation.pimsim_nn import (  # noqa: E402
+    export_raptor_latency_artifact,
+    parse_pimsim_nn_metrics,
+)
 from raptor_validation.validate_one import (  # noqa: E402
     STAGE_COLORS,
     build_dump_ranges,
@@ -70,7 +75,12 @@ def load_pimcomp_exporter():
     module = importlib.util.module_from_spec(spec)
     assert spec is not None and spec.loader is not None
     sys.modules.setdefault("cv2", types.ModuleType("cv2"))
-    spec.loader.exec_module(module)
+    write_bytecode = sys.dont_write_bytecode
+    sys.dont_write_bytecode = True
+    try:
+        spec.loader.exec_module(module)
+    finally:
+        sys.dont_write_bytecode = write_bytecode
     return module
 
 
@@ -183,37 +193,21 @@ def run_logged(
     return proc.stdout
 
 
-def remove_tree(path: Path) -> None:
-    if not path.exists() and not path.is_symlink():
-        return
-    if path.is_symlink() or path.is_file():
-        path.unlink()
-        return
-    while True:
-        children = list(path.iterdir())
-        if not children:
-            break
-        for child in children:
-            remove_tree(child)
-    path.rmdir()
-
-
 def load_model_inputs(model_path: Path, seed: int):
     inputs_desc, outputs_desc = onnx_io(model_path)
     arrays_in_order, _ = gen_random_inputs(inputs_desc, seed=seed)
-    return inputs_desc, outputs_desc, arrays_in_order, arrays_in_order
+    return inputs_desc, outputs_desc, arrays_in_order
 
 
 def load_saved_inputs(
-    model_path: Path,
     inputs_desc: list[tuple[int, str, int, list[int]]],
     inputs_dir: Path,
-) -> tuple[list[np.ndarray], list[np.ndarray]]:
+) -> list[np.ndarray]:
     arrays = []
     for idx, name, elem_type, shape in inputs_desc:
         array = np.loadtxt(inputs_dir / f"in{idx}.csv", delimiter=",", dtype=_ONNX_TO_NP[elem_type]).reshape(shape)
         arrays.append(array)
-    return arrays, arrays
+    return arrays
 
 
 def prepare_pimcomp_model(model_path: Path, out_dir: Path) -> Path:
@@ -328,7 +322,7 @@ def compile_reference(
     runner_base = runner_dir / stem
 
     run_logged(
-        "Reference Emit ONNX IR",
+        "Compile Reference ONNX IR",
         [str(args.raptor_path), str(model_path), "-o", str(onnx_ir_base), "--EmitONNXIR",
          "--mlir-elide-elementsattrs-if-larger=16", "--enable-conv-opt-pass=false"],
         cwd=REPO,
@@ -337,7 +331,7 @@ def compile_reference(
         stage="Compile ONNX",
     )
     run_logged(
-        "Reference Native Compile",
+        "Compile Reference Native",
         [str(args.raptor_path), "-O3", str(model_path), "-o", str(runner_base)],
         cwd=REPO,
         timeout_sec=args.timeout_seconds,
@@ -416,13 +410,18 @@ def compile_raptor_target(
         f"--crossbar-size={hardware['crossbar_size']}",
         f"--crossbar-count={hardware['crossbar_count']}",
         f"--core-count={hardware['core_count']}",
+        f"--pim-target-config={args.pimcomp_config}",
         "--pim-emit-json",
         *args.raptor_extra_arg,
     ]
     print_step("Compile Raptor PIM", cmd, REPO, "Compile PIM")
     start = time.perf_counter()
     command = shell_join(cmd)
-    raptor_extra_args = ["--pim-emit-json", *args.raptor_extra_arg]
+    raptor_extra_args = [
+        f"--pim-target-config={args.pimcomp_config}",
+        "--pim-emit-json",
+        *args.raptor_extra_arg,
+    ]
     try:
         timings = compile_with_raptor(
             model_path,
@@ -459,7 +458,7 @@ def compile_raptor_target(
     return out_dir / "pim", timings
 
 
-def run_rust_validation(
+def run_functional_validation(
     label: str,
     pim_dir: Path,
     config_path: Path,
@@ -510,7 +509,7 @@ def run_rust_validation(
 
 def copy_pimcomp_outputs(source_dir: Path, out_dir: Path):
     out_dir.mkdir(parents=True, exist_ok=True)
-    for name in ("SimulationInfo.gz", "VerificationInfo.json", "MappingResult.txt"):
+    for name in PIMCOMP_OUTPUT_FILES:
         shutil.copy2(source_dir / name, out_dir / name)
 
 
@@ -527,7 +526,7 @@ def compile_pimcomp(
         shutil.copy2(args.pimcomp_config, runtime_config)
     pimcomp_output_dir = out_dir / "output"
     pimcomp_output_dir.mkdir(parents=True, exist_ok=True)
-    for name in ("SimulationInfo.gz", "VerificationInfo.json", "MappingResult.txt"):
+    for name in PIMCOMP_OUTPUT_FILES:
         (pimcomp_output_dir / name).unlink(missing_ok=True)
     model_name = args.pimcomp_model_name or f"compare_{model_path.stem}"
     frontend_json = frontend_json_dir / f"{model_name}.json"
@@ -540,7 +539,7 @@ def compile_pimcomp(
         str(frontend_json),
     ]
     run_logged(
-        "PIMCOMP Frontend",
+        "Compile PIMCOMP Frontend",
         frontend_cmd,
         cwd=args.pimcomp_dir / "frontend",
         timeout_sec=args.timeout_seconds,
@@ -556,20 +555,20 @@ def compile_pimcomp(
         "-s=YES",
     ]
     run_logged(
-        "PIMCOMP Backend",
+        "Compile PIMCOMP Backend",
         backend_cmd,
         cwd=frontend_json_dir.parent,
         timeout_sec=args.timeout_seconds,
         steps=steps,
         stage="Compile PIM",
     )
-    remove_tree(frontend_json_dir.parent)
+    shutil.rmtree(frontend_json_dir.parent)
     return pimcomp_output_dir / "VerificationInfo.json", pimcomp_output_dir / "SimulationInfo.gz"
 
 
 def export_pimcomp_for_pimsim_nn(simulation_info: Path, output_dir: Path) -> Path:
     if output_dir.exists():
-        remove_tree(output_dir)
+        shutil.rmtree(output_dir)
     with gzip.open(simulation_info, "rt", encoding="utf-8") as f:
         sim_info = json.load(f)
 
@@ -598,7 +597,7 @@ def export_pimcomp_for_pimsim_nn(simulation_info: Path, output_dir: Path) -> Pat
 
     for core_idx in core_indices:
         core_key = f"core{core_idx}"
-        instructions = sim_info.get(core_key, []) or [{"op": "lldi", "rd": 0, "imm": 0, "len": 0}]
+        instructions = sim_info.get(core_key, [])
         with open(output_dir / f"core_{core_idx}.json", "w", encoding="utf-8") as f:
             json.dump(instructions, f, separators=(",", ":"))
             f.write("\n")
@@ -622,7 +621,7 @@ def export_pimcomp_for_rust(
     if len(runtime_inputs) != 1:
         raise ValueError("PIMCOMP export currently requires exactly one runtime input tensor")
     if output_dir.exists():
-        remove_tree(output_dir)
+        shutil.rmtree(output_dir)
     exporter = load_pimcomp_exporter()
     with open(verification_info, "r", encoding="utf-8") as f:
         final_info = json.load(f)
@@ -727,7 +726,7 @@ def export_pimcomp_for_rust(
         for sim_inst in sim_info.get(core_name, []) or []:
             op = sim_inst["op"]
             if op == "setbw":
-                instructions.append(sim_inst)
+                instructions.append({"op": "setbw", "ibiw": 32, "obiw": 32})
                 continue
             if op == "sldi":
                 translated = {"op": "sldi", "rd": sim_inst["rd"], "imm": exporter.byte_offset(sim_inst["imm"])}
@@ -782,10 +781,12 @@ def export_pimcomp_for_rust(
                         "offset": sim_inst["offset"],
                     }
                 )
-            elif op in ("lmv", "vvadd", "vvmul", "vvmax", "vrelu"):
+            elif op == "lmv":
                 translated = dict(sim_inst)
                 translated["len"] = exporter.byte_offset(sim_inst["len"])
                 instructions.append(translated)
+            elif op in ("vvadd", "vvmul", "vvmax", "vrelu"):
+                instructions.append(sim_inst)
             elif op in ("send", "recv"):
                 translated = dict(sim_inst)
                 translated["size"] = exporter.byte_offset(sim_inst["size"])
@@ -822,24 +823,6 @@ def export_pimcomp_for_rust(
     return output_dir
 
 
-def parse_pimsim_nn_report(output: str) -> dict[str, float | int | str]:
-    patterns = {
-        "output_count": r"output count:\s+([0-9]+)\s+samples",
-        "throughput": r"throughput:\s+([0-9.eE+-]+)\s+samples/s",
-        "average_latency_ms": r"average latency:\s+([0-9.eE+-]+)\s+ms",
-        "latency_ms": r"latency:\s+([0-9.eE+-]+)\s+ms",
-        "average_power_mw": r"average power:\s+([0-9.eE+-]+)\s+mW",
-        "average_energy_pj": r"average energy:\s+([0-9.eE+-]+)\s+pJ/it",
-    }
-    result: dict[str, float | int | str] = {"raw_output": output}
-    for key, pattern in patterns.items():
-        match = re.search(pattern, output)
-        if match:
-            value = match.group(1)
-            result[key] = int(value) if key == "output_count" else float(value)
-    return result
-
-
 def run_pimsim_nn(
     label: str,
     inst_path: Path,
@@ -861,7 +844,7 @@ def run_pimsim_nn(
         steps=steps,
         stage="Run Non-functional Simulation",
     )
-    return parse_pimsim_nn_report(output)
+    return parse_pimsim_nn_metrics(output)
 
 
 def parse_raptor_instructions(pim_dir: Path) -> dict[str, Any]:
@@ -1069,7 +1052,7 @@ def write_report(
 
     lines.extend(
         [
-            "## Semantic Validation",
+            "## Functional Validation",
             "",
             f"- Raptor via `pim-simulator`: `{validation_status(raptor_validation)}`",
             f"- PIMCOMP via exported `pim-simulator`: `{validation_status(pimcomp_validation)}`",
@@ -1268,6 +1251,7 @@ def main():
     runner_path: Path | None = None
     reference_dir: Path | None = None
     raptor_pim_dir: Path | None = None
+    raptor_pimsim_dir: Path | None = None
     raptor_pass_timings: dict[str, float] = {}
     verification_info: Path | None = None
     simulation_info: Path | None = None
@@ -1289,7 +1273,8 @@ def main():
 
     model_io = try_stage(failures, "Load model inputs", load_model_inputs, model_path, args.seed)
     if model_io is not None:
-        inputs_desc, outputs_desc, arrays_in_order, runtime_inputs = model_io
+        inputs_desc, outputs_desc, arrays_in_order = model_io
+        runtime_inputs = arrays_in_order
 
     if reuse_raptor and model_io is not None:
         reuse_report_path = args.reuse_raptor_report.resolve()
@@ -1300,11 +1285,11 @@ def main():
             raise ValueError(f"Reused Raptor hardware differs: {reused_hardware} != {hardware}")
         reference_dir = Path(reused["paths"]["reference_outputs"])
         raptor_pim_dir = Path(reused["paths"]["raptor_pim"])
-        arrays_in_order, runtime_inputs = load_saved_inputs(
-            model_path,
+        arrays_in_order = load_saved_inputs(
             inputs_desc,
             reference_dir.parent / "inputs",
         )
+        runtime_inputs = arrays_in_order
         raptor_validation = CompareResult(**reused["raptor_validation"])
         raptor_perf = reused["raptor_performance"]
         raptor_instr = reused["raptor_instruction_summary"]
@@ -1392,9 +1377,9 @@ def main():
         if wrote_inputs and reference_dir is not None and outputs_desc:
             validation = try_stage(
                 failures,
-                "Rust Validation Raptor",
-                run_rust_validation,
-                "Rust Validation Raptor",
+                "Functional Validation Raptor",
+                run_functional_validation,
+                "Functional Validation Raptor",
                 raptor_pim_dir,
                 raptor_pim_dir / "config.json",
                 out_dir / "simulation/out.bin",
@@ -1451,7 +1436,7 @@ def main():
     if verification_info is not None and simulation_info is not None and model_io is not None:
         exported = try_stage(
             failures,
-            "Export PIMCOMP for Rust",
+            "Export PIMCOMP for Functional Validation",
             export_pimcomp_for_rust,
             pimcomp_model_path,
             verification_info,
@@ -1464,22 +1449,22 @@ def main():
     elif verification_info is None or simulation_info is None:
         record_failure(
             failures,
-            "Export PIMCOMP for Rust",
-            "PIMCOMP Rust export failed because PIMCOMP did not produce VerificationInfo.json and SimulationInfo.gz.",
+            "Export PIMCOMP for Functional Validation",
+            "PIMCOMP functional export failed because PIMCOMP did not produce VerificationInfo.json and SimulationInfo.gz.",
         )
     else:
         record_failure(
             failures,
-            "Export PIMCOMP for Rust",
-            "PIMCOMP Rust export failed because model inputs are not available.",
+            "Export PIMCOMP for Functional Validation",
+            "PIMCOMP functional export failed because model inputs are not available.",
         )
 
     if pimcomp_export_dir is not None and reference_dir is not None and outputs_desc:
         validation = try_stage(
             failures,
-            "Rust Validation PIMCOMP",
-            run_rust_validation,
-            "Rust Validation PIMCOMP",
+            "Functional Validation PIMCOMP",
+            run_functional_validation,
+            "Functional Validation PIMCOMP",
             pimcomp_export_dir,
             pimcomp_export_dir / "config.json",
             out_dir / "simulation/pimcomp.out.bin",
@@ -1491,7 +1476,7 @@ def main():
         )
         pimcomp_validation = validation if validation is not None else failed_validation("PIMCOMP validation failed")
     elif pimcomp_export_dir is None:
-        pimcomp_validation = failed_validation("PIMCOMP Rust export is not available")
+        pimcomp_validation = failed_validation("PIMCOMP functional export is not available")
     elif reference_dir is None:
         pimcomp_validation = failed_validation("Reference outputs are not available")
     else:
@@ -1524,17 +1509,27 @@ def main():
         pimcomp_perf = skipped_perf("pimsim-nn config is not available")
     else:
         if not reuse_raptor and raptor_pim_dir is not None:
-            perf = try_stage(
+            raptor_pimsim_dir = try_stage(
                 failures,
-                "pimsim-nn Raptor",
-                run_pimsim_nn,
-                "pimsim-nn Raptor",
+                "Export Raptor for pimsim-nn",
+                export_raptor_latency_artifact,
                 raptor_pim_dir,
-                pimsim_config,
-                steps,
-                args,
+                out_dir / "raptor/pimsim_nn",
             )
-            raptor_perf = perf if perf is not None else failed_perf("pimsim-nn Raptor failed")
+            if raptor_pimsim_dir is not None:
+                perf = try_stage(
+                    failures,
+                    "Non-Functional Simulation Raptor",
+                    run_pimsim_nn,
+                    "Non-Functional Simulation Raptor",
+                    raptor_pimsim_dir,
+                    pimsim_config,
+                    steps,
+                    args,
+                )
+                raptor_perf = perf if perf is not None else failed_perf("pimsim-nn Raptor failed")
+            else:
+                raptor_perf = failed_perf("Raptor pimsim-nn export failed")
         elif not reuse_raptor:
             raptor_perf = skipped_perf("Raptor PIM directory is not available")
 
@@ -1549,9 +1544,9 @@ def main():
             if pimcomp_pimsim_dir is not None:
                 perf = try_stage(
                     failures,
-                    "pimsim-nn PIMCOMP",
+                    "Non-Functional Simulation PIMCOMP",
                     run_pimsim_nn,
-                    "pimsim-nn PIMCOMP",
+                    "Non-Functional Simulation PIMCOMP",
                     pimcomp_pimsim_dir,
                     pimsim_config,
                     steps,
@@ -1613,6 +1608,7 @@ def main():
         "paths": {
             "reference_outputs": optional_path(reference_dir),
             "raptor_pim": optional_path(raptor_pim_dir),
+            "raptor_pimsim_nn": optional_path(raptor_pimsim_dir),
             "pimcomp_simulation_info": optional_path(simulation_info),
             "pimcomp_exported_pim": optional_path(pimcomp_export_dir),
             "pimsim_config": optional_path(pimsim_config),
@@ -1625,11 +1621,11 @@ def main():
         f.write("\n")
 
     failed_steps = any(step.status != "passed" for step in steps)
-    semantic_failure = any(
+    functional_failure = any(
         result.status == "done" and not result.passed
         for result in (raptor_validation, pimcomp_validation)
     )
-    failed = bool(failures or failed_steps or semantic_failure)
+    failed = bool(failures or failed_steps or functional_failure)
     result = "FAIL" if args.fail_on_error and failed else "DONE" if failed else "PASS"
     color = Fore.RED if result == "FAIL" else Fore.YELLOW if result == "DONE" else Fore.GREEN
     print("\n" + Style.BRIGHT + f"Result: {color}{result}" + Style.RESET_ALL)

validation/tools/pimcomp/run_pimcomp_paper_latency.py

@@ -0,0 +1,219 @@
+#!/usr/bin/env python3
+from __future__ import annotations
+
+import argparse
+import csv
+import json
+import shlex
+import subprocess
+import sys
+from pathlib import Path
+
+from colorama import Fore, Style
+
+
+REPO = Path(__file__).resolve().parents[3]
+SUITE = REPO / "validation/networks/pimcomp_models"
+sys.path.insert(0, str(REPO / "validation"))
+
+from raptor_validation.pimsim_nn import parse_pimsim_nn_metrics  # noqa: E402
+from raptor_validation.validate_one import STAGE_COLORS  # noqa: E402
+
+PIMCOMP_SOURCE = REPO / "third_party/PIMCOMP-NN"
+PIMCOMP_CONFIG = REPO / "validation/pimsim_configs/pimcomp/arch-a/latency_config.json"
+COMPARE = Path(__file__).resolve().with_name("compare_raptor_pimcomp.py")
+MODELS = {
+    "vgg8": SUITE / "vgg8/vgg8-mnist-reconstructed.onnx",
+    "resnet18": SUITE / "resnet18/resnet18-v1-7.onnx",
+    "resnet34": SUITE / "resnet34/resnet34-v1-7.onnx",
+    "googlenet": SUITE / "googlenet/googlenet-12-latency.onnx",
+}
+
+
+def result_dir(root: Path | None, name: str) -> Path:
+    return root / name if root is not None else MODELS[name].parent
+
+
+def write_results_csv(root: Path | None) -> Path:
+    output = (root or SUITE) / "results.csv"
+    fields = (
+        "model",
+        "raptor_latency_ms",
+        "pimcomp_latency_ms",
+        "raptor_energy_pj",
+        "pimcomp_energy_pj",
+        "faster_compiler",
+        "speedup",
+    )
+    rows = []
+    for name in MODELS:
+        report_path = result_dir(root, name) / "pimcomp/comparison_report.json"
+        if not report_path.exists():
+            continue
+        report = json.loads(report_path.read_text(encoding="utf-8"))
+        raptor = report.get("raptor_performance") or {}
+        pimcomp = report.get("pimcomp_performance") or {}
+        raptor_latency = raptor.get("latency_ms")
+        pimcomp_latency = pimcomp.get("latency_ms")
+        if raptor_latency is None or pimcomp_latency is None:
+            continue
+        raptor_energy = (raptor.get("average_energy_pj")
+                          or parse_pimsim_nn_metrics(raptor.get("raw_output", "")).get("average_energy_pj"))
+        pimcomp_energy = (pimcomp.get("average_energy_pj")
+                           or parse_pimsim_nn_metrics(pimcomp.get("raw_output", "")).get("average_energy_pj"))
+        faster = "raptor" if raptor_latency < pimcomp_latency else "pimcomp"
+        rows.append({
+            "model": name,
+            "raptor_latency_ms": f"{raptor_latency:.6f}",
+            "pimcomp_latency_ms": f"{pimcomp_latency:.6f}",
+            "raptor_energy_pj": "" if raptor_energy is None else f"{raptor_energy:.6f}",
+            "pimcomp_energy_pj": "" if pimcomp_energy is None else f"{pimcomp_energy:.6f}",
+            "faster_compiler": faster,
+            "speedup": f"{max(raptor_latency, pimcomp_latency) / min(raptor_latency, pimcomp_latency):.2f}",
+        })
+    with open(output, "w", newline="", encoding="utf-8") as f:
+        writer = csv.DictWriter(f, fieldnames=fields, lineterminator="\n")
+        writer.writeheader()
+        writer.writerows(rows)
+    return output
+
+
+def print_stage(title: str, color: str) -> None:
+    print("\n" + Style.BRIGHT + color + f"[{title}]" + Style.RESET_ALL, flush=True)
+
+
+def run(command: list[str], *, dry_run: bool, check: bool = True) -> int:
+    print(f"  cwd: {REPO}", flush=True)
+    print(f"  $ {shlex.join(command)}", flush=True)
+    if dry_run:
+        return 0
+    return subprocess.run(command, cwd=REPO, check=check).returncode
+
+
+def validate_pimcomp_source() -> None:
+    header = PIMCOMP_SOURCE / "backend/GeneticAlgorithm.h"
+    source = header.read_text(encoding="utf-8")
+    for setting in ("int population_num = 200;", "int max_iteration = 1000;"):
+        if setting not in source:
+            raise RuntimeError(f"PIMCOMP paper setting is missing: {setting}")
+
+
+def comparison_command(model: Path, result_dir: Path, timeout: float) -> list[str]:
+    return [
+        sys.executable,
+        str(COMPARE),
+        "--model",
+        str(model),
+        "--out-dir",
+        str(result_dir),
+        "--pimcomp-dir",
+        str(PIMCOMP_SOURCE),
+        "--pimcomp-config",
+        str(PIMCOMP_CONFIG),
+        "--core-count",
+        "168",
+        "--crossbar-count",
+        "96",
+        "--crossbar-size",
+        "128",
+        "--mesh-rows",
+        "12",
+        "--mesh-cols",
+        "14",
+        "--pimsim-mode",
+        "latency",
+        "--pimcomp-pipeline",
+        "element",
+        "--pimcomp-replication",
+        "GA",
+        "--timeout-seconds",
+        str(timeout),
+        "--fail-on-error",
+    ]
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(
+        description="Reproduce the serial Arch-A latency comparison from the PIMCOMP paper."
+    )
+    parser.add_argument(
+        "--out-dir",
+        type=Path,
+        help="Result root (default: artifacts beside each model under validation/).",
+    )
+    parser.add_argument("--models", nargs="+", choices=MODELS, default=list(MODELS))
+    parser.add_argument("--timeout-seconds", type=float, default=3600.0)
+    parser.add_argument(
+        "--resume",
+        action="store_true",
+        help="Skip models with a completed JSON report.",
+    )
+    parser.add_argument("--dry-run", action="store_true", help="Print commands without modifying files.")
+    args = parser.parse_args()
+
+    out_dir = args.out_dir.resolve() if args.out_dir is not None else None
+
+    missing = [str(MODELS[name]) for name in args.models if not MODELS[name].exists()]
+    if missing:
+        parser.error(f"missing model(s): {', '.join(missing)}")
+
+    validate_pimcomp_source()
+    if out_dir is not None and not args.dry_run:
+        out_dir.mkdir(parents=True, exist_ok=True)
+
+    print(Style.BRIGHT + f"Found {len(args.models)} PIMCOMP model(s) to compare." + Style.RESET_ALL)
+    print(f"Results root: {out_dir or SUITE}")
+    print("=" * 72)
+
+    print_stage("Build Raptor", STAGE_COLORS["Build Runner"])
+    run(["cmake", "--build", str(REPO / "build_release")], dry_run=args.dry_run)
+
+    print_stage("Build PIMCOMP", STAGE_COLORS["Build Runner"])
+    run(
+        ["cmake", "--build", str(PIMCOMP_SOURCE / "build"), "--target", "PIMCOMP-NN"],
+        dry_run=args.dry_run,
+    )
+
+    failed = []
+    for index, name in enumerate(args.models, start=1):
+        model_result_dir = result_dir(out_dir, name)
+        print(
+            "\n" + Fore.CYAN + f"[{index}/{len(args.models)}]" + Style.RESET_ALL
+            + f" {Style.BRIGHT}Comparing {name}{Style.RESET_ALL}",
+            flush=True,
+        )
+        if args.resume and (model_result_dir / "pimcomp/comparison_report.json").exists():
+            print(
+                Fore.YELLOW + "  Completed report exists; skipping" + Style.RESET_ALL,
+                flush=True,
+            )
+            continue
+        returncode = run(
+            comparison_command(MODELS[name], model_result_dir, args.timeout_seconds),
+            dry_run=args.dry_run,
+            check=False,
+        )
+        if returncode:
+            failed.append(name)
+
+    if args.dry_run:
+        return 1 if failed else 0
+
+    results_path = write_results_csv(out_dir)
+    print_stage("Results", STAGE_COLORS["Compare Outputs"])
+    print(results_path.read_text(encoding="utf-8"), end="")
+    print("\n" + Style.BRIGHT + Fore.CYAN + "Summary" + Style.RESET_ALL)
+    print(Style.BRIGHT + f"Passed: {len(args.models) - len(failed)}" + Style.RESET_ALL)
+    print(Style.BRIGHT + f"Failed: {len(failed)}" + Style.RESET_ALL)
+    print(Style.BRIGHT + f"Results: {results_path}" + Style.RESET_ALL)
+    if failed:
+        print(
+            Fore.RED + f"Failed comparisons: {', '.join(failed)}" + Style.RESET_ALL,
+            file=sys.stderr,
+        )
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

validation/tools/run_pimcomp_paper_latency.py

@@ -1,148 +0,0 @@
-#!/usr/bin/env python3
-from __future__ import annotations
-
-import argparse
-import shlex
-import subprocess
-import sys
-from pathlib import Path
-
-from colorama import Fore, Style
-
-
-REPO = Path(__file__).resolve().parents[2]
-SUITE = REPO / "validation/networks/pimcomp_models"
-PIMCOMP_SOURCE = REPO / "third_party/PIMCOMP-NN"
-PIMCOMP_CONFIG = REPO / "validation/pimsim_configs/pimcomp/arch-a/latency_config.json"
-COMPARE = REPO / "validation/tools/compare_raptor_pimcomp.py"
-MODELS = {
-    "vgg8": SUITE / "vgg8/vgg8-mnist-reconstructed.onnx",
-    "resnet18": SUITE / "resnet18/resnet18-v1-7.onnx",
-    "resnet34": SUITE / "resnet34/resnet34-v1-7.onnx",
-    "googlenet": SUITE / "googlenet/googlenet-12-no-softmax.onnx",
-}
-
-
-def run(command: list[str], *, dry_run: bool, check: bool = True) -> int:
-    print(Fore.CYAN + "$ " + Style.RESET_ALL + shlex.join(command), flush=True)
-    if dry_run:
-        return 0
-    return subprocess.run(command, cwd=REPO, check=check).returncode
-
-
-def validate_pimcomp_source() -> None:
-    header = PIMCOMP_SOURCE / "backend/GeneticAlgorithm.h"
-    source = header.read_text(encoding="utf-8")
-    for setting in ("int population_num = 200;", "int max_iteration = 1000;"):
-        if setting not in source:
-            raise RuntimeError(f"PIMCOMP paper setting is missing: {setting}")
-
-
-def comparison_command(model: Path, result_dir: Path, timeout: float) -> list[str]:
-    return [
-        sys.executable,
-        str(COMPARE),
-        "--model",
-        str(model),
-        "--out-dir",
-        str(result_dir),
-        "--pimcomp-dir",
-        str(PIMCOMP_SOURCE),
-        "--pimcomp-config",
-        str(PIMCOMP_CONFIG),
-        "--core-count",
-        "168",
-        "--crossbar-count",
-        "96",
-        "--crossbar-size",
-        "128",
-        "--mesh-rows",
-        "12",
-        "--mesh-cols",
-        "14",
-        "--pimsim-mode",
-        "latency",
-        "--pimcomp-pipeline",
-        "element",
-        "--pimcomp-replication",
-        "GA",
-        "--timeout-seconds",
-        str(timeout),
-        "--fail-on-error",
-    ]
-
-
-def main() -> int:
-    parser = argparse.ArgumentParser(
-        description="Reproduce the serial Arch-A latency comparison from the PIMCOMP paper."
-    )
-    parser.add_argument(
-        "--out-dir",
-        type=Path,
-        help="Result root (default: artifacts beside each model under validation/).",
-    )
-    parser.add_argument("--models", nargs="+", choices=MODELS, default=list(MODELS))
-    parser.add_argument("--timeout-seconds", type=float, default=3600.0)
-    parser.add_argument(
-        "--resume",
-        action="store_true",
-        help="Skip models with a completed JSON report.",
-    )
-    parser.add_argument("--dry-run", action="store_true", help="Print commands without modifying files.")
-    args = parser.parse_args()
-
-    out_dir = args.out_dir.resolve() if args.out_dir is not None else None
-
-    missing = [str(MODELS[name]) for name in args.models if not MODELS[name].exists()]
-    if missing:
-        parser.error(f"missing model(s): {', '.join(missing)}")
-
-    validate_pimcomp_source()
-    if out_dir is not None and not args.dry_run:
-        out_dir.mkdir(parents=True, exist_ok=True)
-
-    run(["cmake", "--build", str(REPO / "build_release")], dry_run=args.dry_run)
-    run(
-        ["cmake", "--build", str(PIMCOMP_SOURCE / "build"), "--target", "PIMCOMP-NN"],
-        dry_run=args.dry_run,
-    )
-
-    failed = []
-    for name in args.models:
-        result_dir = out_dir / name if out_dir is not None else MODELS[name].parent
-        if args.resume and (result_dir / "pimcomp/comparison_report.json").exists():
-            print(
-                Fore.YELLOW + f"[{name}] completed report exists; skipping" + Style.RESET_ALL,
-                flush=True,
-            )
-            continue
-        print(
-            "\n" + Fore.CYAN + f"[{name}]" + Style.RESET_ALL
-            + f" {Style.BRIGHT}Arch-A latency comparison{Style.RESET_ALL}",
-            flush=True,
-        )
-        returncode = run(
-            comparison_command(MODELS[name], result_dir, args.timeout_seconds),
-            dry_run=args.dry_run,
-            check=False,
-        )
-        if returncode:
-            failed.append(name)
-
-    if failed:
-        print(
-            "\n" + Style.BRIGHT + Fore.RED + "Result: FAIL" + Style.RESET_ALL,
-            file=sys.stderr,
-        )
-        print(
-            Fore.RED + f"Failed comparisons: {', '.join(failed)}" + Style.RESET_ALL,
-            file=sys.stderr,
-        )
-        return 1
-    if not args.dry_run:
-        print("\n" + Style.BRIGHT + f"Result: {Fore.GREEN}PASS" + Style.RESET_ALL)
-    return 0
-
-
-if __name__ == "__main__":
-    raise SystemExit(main())