add non-functional validation

refactors

validation/tools/classification_local_image_validation.py

@@ -15,8 +15,8 @@ REPO_ROOT = VALIDATION_DIR.parent
 if str(VALIDATION_DIR) not in sys.path:
     sys.path.insert(0, str(VALIDATION_DIR))
 
-from onnx_utils import _ONNX_TO_NP, onnx_io, write_inputs_to_memory_bin
-from validate_one import (
+from raptor_validation.onnx_utils import _ONNX_TO_NP, onnx_io, write_inputs_to_memory_bin
+from raptor_validation.validate_one import (
     MODE_COMPILE_ONLY,
     build_dump_ranges,
     parse_pim_simulator_outputs,
@@ -59,6 +59,15 @@ def ensure_local_artifacts(args, model_path: Path):
         crossbar_size=args.crossbar_size,
         crossbar_count=args.crossbar_count,
         core_count=args.core_count,
+        raptor_extra_args=[],
+        pimsim_nn_build_dir=None,
+        pimsim_config_path=None,
+        threshold=1e-3,
+        rtol=1e-5,
+        seed=0,
+        reporter=None,
+        model_index=1,
+        model_total=1,
         command_timeout_seconds=args.command_timeout_seconds,
         mode=MODE_COMPILE_ONLY,
         verbose=args.verbose,

validation/tools/compare_raptor_pimcomp.py

@@ -25,12 +25,19 @@ import onnx
 
 REPO = Path(__file__).resolve().parents[2]
 VALIDATION_DIR = REPO / "validation"
+PIMSIM_CONFIG_DIR = VALIDATION_DIR / "pimsim_configs/pimcomp"
 sys.path.insert(0, str(VALIDATION_DIR))
 
-from gen_network_runner import gen_network_runner  # noqa: E402
-from onnx_utils import _ONNX_TO_NP, gen_random_inputs, onnx_io, save_inputs_to_files, write_inputs_to_memory_bin  # noqa: E402
-from validate_one import build_dump_ranges, parse_pim_simulator_outputs  # noqa: E402
-from raptor import compile_with_raptor  # noqa: E402
+from raptor_validation.gen_network_runner import gen_network_runner  # noqa: E402
+from raptor_validation.onnx_utils import (  # noqa: E402
+    _ONNX_TO_NP,
+    gen_random_inputs,
+    onnx_io,
+    save_inputs_to_files,
+    write_inputs_to_memory_bin,
+)
+from raptor_validation.raptor import compile_with_raptor  # noqa: E402
+from raptor_validation.validate_one import build_dump_ranges, parse_pim_simulator_outputs  # noqa: E402
 
 
 @dataclass
@@ -62,15 +69,6 @@ def load_pimcomp_exporter():
     return module
 
 
-def load_mesh_builder():
-    path = REPO / "validation/pimsim-configs/generate_mesh_config.py"
-    spec = importlib.util.spec_from_file_location("mesh_builder", path)
-    module = importlib.util.module_from_spec(spec)
-    assert spec is not None and spec.loader is not None
-    spec.loader.exec_module(module)
-    return module
-
-
 def shell_join(cmd: list[str]) -> str:
     return shlex.join(str(arg) for arg in cmd)
 
@@ -278,41 +276,26 @@ def load_effective_hardware(args: argparse.Namespace) -> dict[str, int]:
     return hardware
 
 
-def write_pimsim_config(args: argparse.Namespace, out_dir: Path, hardware: dict[str, int]) -> Path:
-    mesh_builder = load_mesh_builder()
-    with open(args.pimcomp_dir / "config.json", "r", encoding="utf-8") as f:
-        config = json.load(f)
-    config["chip_config"]["core_config"].setdefault("rob_size", 1)
-    config["chip_config"]["core_config"]["matrix_config"]["xbar_array_count"] = hardware["crossbar_count"]
-    config["chip_config"]["core_config"]["matrix_config"]["xbar_size"] = [
-        hardware["crossbar_size"],
-        hardware["crossbar_size"],
-    ]
-    config["chip_config"]["network_config"]["layout"] = [
-        hardware["mesh_rows"],
-        hardware["mesh_cols"],
-    ]
-    config["chip_config"]["network_config"]["net_config_file_path"] = f"network_mesh_{hardware['core_count']}.json"
-    config["chip_config"]["core_cnt"] = hardware["core_count"]
-    config["sim_config"]["sim_mode"] = 1 if args.pimsim_mode == "latency" else 0
-    config["sim_config"]["sim_time"] = args.pimsim_time_ms
-    out_dir.mkdir(parents=True, exist_ok=True)
-    config_path = out_dir / f"{args.pimsim_mode}_config.json"
-    network_path = out_dir / f"network_mesh_{hardware['core_count']}.json"
-    with open(config_path, "w", encoding="utf-8") as f:
-        json.dump(config, f, indent=2)
-        f.write("\n")
-    with open(network_path, "w", encoding="utf-8") as f:
-        json.dump(
-            mesh_builder.build_network(
-                hardware["core_count"],
-                (hardware["mesh_rows"], hardware["mesh_cols"]),
-            ),
-            f,
-            separators=(",", ":"),
-        )
-        f.write("\n")
-    return config_path
+def select_pimsim_config(args: argparse.Namespace, hardware: dict[str, int]) -> Path:
+    for path in sorted(PIMSIM_CONFIG_DIR.glob(f"*/{args.pimsim_mode}_config.json")):
+        with open(path, encoding="utf-8") as f:
+            config = json.load(f)
+        chip = config["chip_config"]
+        matrix = chip["core_config"]["matrix_config"]
+        network = chip["network_config"]
+        if (
+            chip["core_cnt"] == hardware["core_count"]
+            and matrix["xbar_array_count"] == hardware["crossbar_count"]
+            and matrix["xbar_size"] == [hardware["crossbar_size"]] * 2
+            and network["layout"] == [hardware["mesh_rows"], hardware["mesh_cols"]]
+            and config["sim_config"]["sim_mode"] == (1 if args.pimsim_mode == "latency" else 0)
+            and config["sim_config"]["sim_time"] == args.pimsim_time_ms
+        ):
+            return path
+    raise ValueError(
+        f"No pre-generated {args.pimsim_mode} pimsim-nn config matches "
+        f"{hardware} with sim_time={args.pimsim_time_ms}"
+    )
 
 
 def compile_reference(
@@ -348,7 +331,14 @@ def compile_reference(
     network_so = runner_base.with_suffix(".so")
 
     print_step("Generate Runner Source")
-    gen_network_runner(model_path, network_so, args.onnx_include_dir, out=runner_dir / "runner.c", verbose=False)
+    gen_network_runner(
+        model_path,
+        network_so,
+        args.onnx_include_dir,
+        entry="run_main_graph",
+        out=runner_dir / "runner.c",
+        verbose=False,
+    )
 
     run_logged(
         "Configure Runner",
@@ -427,6 +417,7 @@ def compile_raptor_target(
             raptor_extra_args=raptor_extra_args,
             cwd=out_dir,
             verbose=args.verbose_raptor_compile,
+            reporter=None,
             timeout_sec=args.timeout_seconds,
         )
     except Exception as exc:
@@ -1453,18 +1444,17 @@ def main():
     else:
         pimcomp_validation = skipped_validation("Output descriptors are not available")
 
-    if hardware["core_count"] > 0:
+    if not args.skip_pimsim_nn and hardware["core_count"] > 0:
         written_config = try_stage(
             failures,
-            "Write pimsim-nn config",
-            write_pimsim_config,
+            "Select pimsim-nn config",
+            select_pimsim_config,
             args,
-            out_dir / "pimsim_config",
             hardware,
         )
         if written_config is not None:
             pimsim_config = written_config
-    else:
+    elif not args.skip_pimsim_nn:
         record_failure(
             failures,
             "Skip pimsim-nn config",

validation/tools/run_pimcomp_paper_latency.py

@@ -48,7 +48,10 @@ def prepare_pimcomp(work_dir: Path) -> None:
     if replacements != 1:
         raise RuntimeError("Could not set PIMCOMP GA max_iteration")
     header.write_text(source, encoding="utf-8")
-    shutil.copy2(SUITE / "configs/arch-a.json", work_dir / "config.json")
+    shutil.copy2(
+        REPO / "validation/pimsim_configs/pimcomp/arch-a/latency_config.json",
+        work_dir / "config.json",
+    )
 
 
 def comparison_command(model: Path, result_dir: Path, pimcomp_dir: Path, timeout: float) -> list[str]:
@@ -79,7 +82,6 @@ def comparison_command(model: Path, result_dir: Path, pimcomp_dir: Path, timeout
         "GA",
         "--timeout-seconds",
         str(timeout),
-        "--fail-on-error",
     ]
 
 

validation/tools/yolo_local_image_validation.py

@@ -21,8 +21,14 @@ if sys.version_info < (3, 10):
         "Run it with a newer interpreter, for example your project venv Python."
     )
 
-from onnx_utils import _ONNX_TO_NP, onnx_io, write_inputs_to_memory_bin
-from validate_one import MODE_COMPILE_ONLY, build_dump_ranges, run_pim_simulator, sanitize_output_name, validate_network
+from raptor_validation.onnx_utils import _ONNX_TO_NP, onnx_io, write_inputs_to_memory_bin
+from raptor_validation.validate_one import (
+    MODE_COMPILE_ONLY,
+    build_dump_ranges,
+    run_pim_simulator,
+    sanitize_output_name,
+    validate_network,
+)
 from yolo_real_image_validation import (
     IMAGE_CASES,
     decode_yolo_output,
@@ -80,6 +86,15 @@ def ensure_local_artifacts(args, network_onnx_path: Path):
         crossbar_size=args.crossbar_size,
         crossbar_count=args.crossbar_count,
         core_count=args.core_count,
+        raptor_extra_args=[],
+        pimsim_nn_build_dir=None,
+        pimsim_config_path=None,
+        threshold=1e-3,
+        rtol=1e-5,
+        seed=0,
+        reporter=None,
+        model_index=1,
+        model_total=1,
         command_timeout_seconds=args.command_timeout_seconds,
         mode=MODE_COMPILE_ONLY,
         verbose=args.verbose,