add non-functional validation

refactors

validation/tools/compare_raptor_pimcomp.py

@@ -25,12 +25,19 @@ import onnx
 
 REPO = Path(__file__).resolve().parents[2]
 VALIDATION_DIR = REPO / "validation"
+PIMSIM_CONFIG_DIR = VALIDATION_DIR / "pimsim_configs/pimcomp"
 sys.path.insert(0, str(VALIDATION_DIR))
 
-from gen_network_runner import gen_network_runner  # noqa: E402
-from onnx_utils import _ONNX_TO_NP, gen_random_inputs, onnx_io, save_inputs_to_files, write_inputs_to_memory_bin  # noqa: E402
-from validate_one import build_dump_ranges, parse_pim_simulator_outputs  # noqa: E402
-from raptor import compile_with_raptor  # noqa: E402
+from raptor_validation.gen_network_runner import gen_network_runner  # noqa: E402
+from raptor_validation.onnx_utils import (  # noqa: E402
+    _ONNX_TO_NP,
+    gen_random_inputs,
+    onnx_io,
+    save_inputs_to_files,
+    write_inputs_to_memory_bin,
+)
+from raptor_validation.raptor import compile_with_raptor  # noqa: E402
+from raptor_validation.validate_one import build_dump_ranges, parse_pim_simulator_outputs  # noqa: E402
 
 
 @dataclass
@@ -62,15 +69,6 @@ def load_pimcomp_exporter():
     return module
 
 
-def load_mesh_builder():
-    path = REPO / "validation/pimsim-configs/generate_mesh_config.py"
-    spec = importlib.util.spec_from_file_location("mesh_builder", path)
-    module = importlib.util.module_from_spec(spec)
-    assert spec is not None and spec.loader is not None
-    spec.loader.exec_module(module)
-    return module
-
-
 def shell_join(cmd: list[str]) -> str:
     return shlex.join(str(arg) for arg in cmd)
 
@@ -278,41 +276,26 @@ def load_effective_hardware(args: argparse.Namespace) -> dict[str, int]:
     return hardware
 
 
-def write_pimsim_config(args: argparse.Namespace, out_dir: Path, hardware: dict[str, int]) -> Path:
-    mesh_builder = load_mesh_builder()
-    with open(args.pimcomp_dir / "config.json", "r", encoding="utf-8") as f:
-        config = json.load(f)
-    config["chip_config"]["core_config"].setdefault("rob_size", 1)
-    config["chip_config"]["core_config"]["matrix_config"]["xbar_array_count"] = hardware["crossbar_count"]
-    config["chip_config"]["core_config"]["matrix_config"]["xbar_size"] = [
-        hardware["crossbar_size"],
-        hardware["crossbar_size"],
-    ]
-    config["chip_config"]["network_config"]["layout"] = [
-        hardware["mesh_rows"],
-        hardware["mesh_cols"],
-    ]
-    config["chip_config"]["network_config"]["net_config_file_path"] = f"network_mesh_{hardware['core_count']}.json"
-    config["chip_config"]["core_cnt"] = hardware["core_count"]
-    config["sim_config"]["sim_mode"] = 1 if args.pimsim_mode == "latency" else 0
-    config["sim_config"]["sim_time"] = args.pimsim_time_ms
-    out_dir.mkdir(parents=True, exist_ok=True)
-    config_path = out_dir / f"{args.pimsim_mode}_config.json"
-    network_path = out_dir / f"network_mesh_{hardware['core_count']}.json"
-    with open(config_path, "w", encoding="utf-8") as f:
-        json.dump(config, f, indent=2)
-        f.write("\n")
-    with open(network_path, "w", encoding="utf-8") as f:
-        json.dump(
-            mesh_builder.build_network(
-                hardware["core_count"],
-                (hardware["mesh_rows"], hardware["mesh_cols"]),
-            ),
-            f,
-            separators=(",", ":"),
-        )
-        f.write("\n")
-    return config_path
+def select_pimsim_config(args: argparse.Namespace, hardware: dict[str, int]) -> Path:
+    for path in sorted(PIMSIM_CONFIG_DIR.glob(f"*/{args.pimsim_mode}_config.json")):
+        with open(path, encoding="utf-8") as f:
+            config = json.load(f)
+        chip = config["chip_config"]
+        matrix = chip["core_config"]["matrix_config"]
+        network = chip["network_config"]
+        if (
+            chip["core_cnt"] == hardware["core_count"]
+            and matrix["xbar_array_count"] == hardware["crossbar_count"]
+            and matrix["xbar_size"] == [hardware["crossbar_size"]] * 2
+            and network["layout"] == [hardware["mesh_rows"], hardware["mesh_cols"]]
+            and config["sim_config"]["sim_mode"] == (1 if args.pimsim_mode == "latency" else 0)
+            and config["sim_config"]["sim_time"] == args.pimsim_time_ms
+        ):
+            return path
+    raise ValueError(
+        f"No pre-generated {args.pimsim_mode} pimsim-nn config matches "
+        f"{hardware} with sim_time={args.pimsim_time_ms}"
+    )
 
 
 def compile_reference(
@@ -348,7 +331,14 @@ def compile_reference(
     network_so = runner_base.with_suffix(".so")
 
     print_step("Generate Runner Source")
-    gen_network_runner(model_path, network_so, args.onnx_include_dir, out=runner_dir / "runner.c", verbose=False)
+    gen_network_runner(
+        model_path,
+        network_so,
+        args.onnx_include_dir,
+        entry="run_main_graph",
+        out=runner_dir / "runner.c",
+        verbose=False,
+    )
 
     run_logged(
         "Configure Runner",
@@ -427,6 +417,7 @@ def compile_raptor_target(
             raptor_extra_args=raptor_extra_args,
             cwd=out_dir,
             verbose=args.verbose_raptor_compile,
+            reporter=None,
             timeout_sec=args.timeout_seconds,
         )
     except Exception as exc:
@@ -1453,18 +1444,17 @@ def main():
     else:
         pimcomp_validation = skipped_validation("Output descriptors are not available")
 
-    if hardware["core_count"] > 0:
+    if not args.skip_pimsim_nn and hardware["core_count"] > 0:
         written_config = try_stage(
             failures,
-            "Write pimsim-nn config",
-            write_pimsim_config,
+            "Select pimsim-nn config",
+            select_pimsim_config,
             args,
-            out_dir / "pimsim_config",
             hardware,
         )
         if written_config is not None:
             pimsim_config = written_config
-    else:
+    elif not args.skip_pimsim_nn:
         record_failure(
             failures,
             "Skip pimsim-nn config",