add ablation study

normalize names and artifact paths

validation/raptor_validation/validate_one.py

@@ -9,6 +9,7 @@ import numpy as np
 from dataclasses import dataclass, field
 from pathlib import Path
 from colorama import Style, Fore
+from .artifacts import artifacts_dir, runner_uses_library
 from .gen_network_runner import gen_network_runner
 from .onnx_utils import (
     _ONNX_TO_NP,
@@ -26,13 +27,13 @@ from .subprocess_utils import run_command_with_reporter
 
 STAGE_TITLES = (
     "Compile ONNX",
-    "Build Runner",
-    "Generate Inputs",
-    "Run Reference",
-    "Compile PIM",
-    "Run Functional Simulation",
-    "Compare Outputs",
-    "Run Non-functional Simulation",
+    "Build runner",
+    "Generate inputs",
+    "Run reference",
+    "Compile Pim",
+    "Run functional simulation",
+    "Compare outputs",
+    "Run non-functional simulation",
 )
 STAGE_COLORS = {
     STAGE_TITLES[0]: Fore.BLUE,
@@ -46,8 +47,7 @@ STAGE_COLORS = {
 }
 STAGE_COUNT = len(STAGE_TITLES)
 GENERATED_DIR_NAMES = (
-    "inputs", "outputs", "pimcomp", "raptor", "runner", "simulation",
-    "throughput_validation",
+    "artifacts",
 )
 
 MODE_FULL = "full"
@@ -58,15 +58,15 @@ MODE_STAGE_TITLES = {
     MODE_FULL: STAGE_TITLES,
     MODE_COMPILE_ONLY: (
         "Compile ONNX",
-        "Build Runner",
-        "Compile PIM",
+        "Build runner",
+        "Compile Pim",
     ),
     MODE_RUN_ONLY: (
-        "Generate Inputs",
-        "Run Reference",
-        "Run Functional Simulation",
-        "Compare Outputs",
-        "Run Non-functional Simulation",
+        "Generate inputs",
+        "Run reference",
+        "Run functional simulation",
+        "Compare outputs",
+        "Run non-functional simulation",
     ),
 }
 
@@ -75,7 +75,7 @@ PIMSIM_FAILED = "ERROR"
 PIMSIM_UNSUPPORTED = "UNSUPPORTED"
 PIMSIM_SKIPPED = "SKIP"
 PIMSIM_NOT_RUN = "-"
-PIMSIM_UNSUPPORTED_VSOFTMAX = "pimsim-nn does not support opcode vsoftmax"
+PIMSIM_UNSUPPORTED_VSOFTMAX = "Pimsim does not support opcode vsoftmax"
 
 
 class PimSimUnsupportedError(RuntimeError):
@@ -338,7 +338,7 @@ def run_pimsim_nn(
         else ("throughput", "average_latency_ms", "average_power_mw", "average_energy_pj")
     )
     if any(name not in metrics for name in required):
-        raise RuntimeError(f"pimsim-nn output did not contain required {execution_mode} metrics")
+        raise RuntimeError(f"Pimsim output did not contain required {execution_mode} metrics")
     return metrics
 
 
@@ -361,6 +361,11 @@ def clean_workspace_artifacts(workspace_dir, model_stem):
     for suffix in (".onnx.mlir", ".so", ".tmp"):
         remove_path(workspace_dir / f"{model_stem}{suffix}")
 
+    for path in workspace_dir.rglob("*.lock"):
+        if path.is_file() or path.is_symlink():
+            path.unlink(missing_ok=True)
+            removed_paths.append(path)
+
     return removed_paths
 
 
@@ -407,32 +412,34 @@ def build_dump_ranges(config_path, outputs_descriptor):
 
 
 def build_pim_simulator_command(
-        pim_dir, output_bin_path, dump_ranges, input_paths, mode="latency",
+        pim_dir, output_bin_path, dump_ranges, input_dir, batch_size, mode="latency",
         batch_output_dir=None):
     if mode not in ("latency", "throughput"):
         raise ValueError(f"unknown simulator mode: {mode}")
-    if not input_paths:
+    if batch_size < 1:
         raise ValueError("simulator requires at least one input")
+    if input_dir is None:
+        raise ValueError("simulator requires an input directory")
     command = [
         "cargo", "run", "--no-default-features", "--release", "--package", "pim-simulator", "--bin", "pim-simulator",
         "--", "-f", str(pim_dir), "-o", str(output_bin_path), "-d", dump_ranges,
-        "--mode", mode, "--batch-size", str(len(input_paths)),
+        "--mode", mode, "--batch-size", str(batch_size),
+        "--input-dir", str(input_dir),
     ]
     if batch_output_dir is not None:
         command += ["--batch-output-dir", str(batch_output_dir)]
-    for path in input_paths:
-        command += ["--input", str(path)]
     return command
 
 
 def run_pim_simulator(
         simulator_dir, pim_dir, output_bin_path, dump_ranges, reporter=None,
-        timeout_sec=None, input_paths=(), mode="latency", batch_output_dir=None):
+        timeout_sec=None, input_dir=None, batch_size=1, mode="latency", batch_output_dir=None):
     command = build_pim_simulator_command(
         pim_dir,
         output_bin_path,
         dump_ranges,
-        input_paths,
+        input_dir,
+        batch_size,
         mode=mode,
         batch_output_dir=batch_output_dir,
     )
@@ -516,7 +523,7 @@ def validate_execution(
         try:
             print_stage(
                 reporter, model_index, model_total, model_name,
-                "Run Functional Simulation", name,
+                "Run functional simulation", name,
             )
             write_inputs_to_memory_bin(
                 pim_dir / "memory.bin", pim_dir / "config.json", input_batch[0])
@@ -526,13 +533,13 @@ def validate_execution(
             run_pim_simulator(
                 simulator_dir, pim_dir, simulation_dir / "out.bin", dump_ranges,
                 reporter=reporter, timeout_sec=command_timeout_seconds,
-                input_paths=input_paths[:batch_size], mode=name,
+                input_dir=input_paths[0].parent, batch_size=batch_size, mode=name,
                 batch_output_dir=output_dir)
             reporter.advance()
 
             print_stage(
                 reporter, model_index, model_total, model_name,
-                "Compare Outputs", name,
+                "Compare outputs", name,
             )
             reporter.suspend()
             try:
@@ -553,7 +560,7 @@ def validate_execution(
 
     print_stage(
         reporter, model_index, model_total, model_name,
-        "Run Non-functional Simulation", name,
+        "Run non-functional simulation", name,
     )
     config_path = execution["pimsim_config"]
     if state["compiled"] and pimsim_nn_build_dir is not None and config_path is not None:
@@ -580,7 +587,7 @@ def validate_execution(
     elif not state["compiled"]:
         state["pimsim_status"] = PIMSIM_NOT_RUN
     else:
-        print_info(reporter, "pimsim-nn non-functional simulation skipped")
+        print_info(reporter, "Pimsim non-functional simulation skipped")
     reporter.advance()
 
 
@@ -609,12 +616,12 @@ def validate_network(network_onnx_path, raptor_path, onnx_include_dir,
     owns_reporter = reporter is None
     reporter = reporter or ProgressReporter(model_total, stages_per_model=len(MODE_STAGE_TITLES[mode]), verbose=verbose)
 
-    workspace_dir = network_onnx_path.parent
+    workspace_dir = artifacts_dir(network_onnx_path.parent)
     raptor_dir = workspace_dir / "raptor"
     runner_dir = workspace_dir / "runner"
     runner_build_dir = runner_dir / "build"
     if mode != MODE_RUN_ONLY:
-        clean_workspace_artifacts(workspace_dir, network_onnx_path.stem)
+        clean_workspace_artifacts(network_onnx_path.parent, network_onnx_path.stem)
         Path.mkdir(raptor_dir, parents=True, exist_ok=True)
         Path.mkdir(runner_build_dir, parents=True, exist_ok=True)
 
@@ -669,7 +676,7 @@ def validate_network(network_onnx_path, raptor_path, onnx_include_dir,
                 print_info(reporter, f"Shared library saved to {network_so_path}")
                 reporter.advance()
 
-                print_stage(reporter, model_index, model_total, network_onnx_path.name, "Build Runner")
+                print_stage(reporter, model_index, model_total, network_onnx_path.name, "Build runner")
                 gen_network_runner(
                     network_onnx_path, network_so_path, onnx_include_dir,
                     entry="run_main_graph", out=runner_dir / "runner.c", verbose=False)
@@ -683,7 +690,10 @@ def validate_network(network_onnx_path, raptor_path, onnx_include_dir,
                 report_validation_failure(reporter, "reference", "compilation", exc)
         else:
             required_paths = (network_so_path, network_mlir_path, runner_path)
-            reference_ready = all(path.exists() for path in required_paths)
+            reference_ready = (
+                all(path.exists() for path in required_paths)
+                and runner_uses_library(runner_path, network_so_path)
+            )
             if not reference_ready:
                 report_validation_failure(reporter, "reference", "artifact lookup", FileNotFoundError(
                     "run-only mode requires the compiled shared library, ONNX MLIR, and runner"))
@@ -696,7 +706,7 @@ def validate_network(network_onnx_path, raptor_path, onnx_include_dir,
                 states[name]["compiled"] = (pim_dir / "config.json").exists()
                 if not states[name]["compiled"]:
                     report_validation_failure(reporter, name, "artifact lookup", FileNotFoundError(
-                        f"run-only mode requires compiled PIM artifacts at {pim_dir}"))
+                        f"run-only mode requires compiled Pim artifacts at {pim_dir}"))
                 else:
                     states[name]["resource_metrics"] = collect_pim_resource_metrics(pim_dir)
                     if name == "latency":
@@ -705,7 +715,7 @@ def validate_network(network_onnx_path, raptor_path, onnx_include_dir,
             try:
                 print_stage(
                     reporter, model_index, model_total, network_onnx_path.name,
-                    "Compile PIM", name,
+                        "Compile Pim", name,
                 )
                 root.mkdir(parents=True, exist_ok=True)
                 started = time.perf_counter()
@@ -724,7 +734,7 @@ def validate_network(network_onnx_path, raptor_path, onnx_include_dir,
                 states[name]["resource_metrics"] = collect_pim_resource_metrics(pim_dir)
                 if name == "latency":
                     resource_metrics = states[name]["resource_metrics"]
-                print_info(reporter, f"PIM artifacts saved to {pim_dir}")
+                print_info(reporter, f"Pim artifacts saved to {pim_dir}")
             except Exception as exc:
                 report_validation_failure(reporter, name, "compilation", exc)
             reporter.advance()
@@ -735,7 +745,7 @@ def validate_network(network_onnx_path, raptor_path, onnx_include_dir,
         else:
             input_batch = input_paths = reference_dirs = outputs_descriptor = None
             try:
-                print_stage(reporter, model_index, model_total, network_onnx_path.name, "Generate Inputs")
+                print_stage(reporter, model_index, model_total, network_onnx_path.name, "Generate inputs")
                 inputs_descriptor, outputs_descriptor = onnx_io(network_onnx_path)
                 first_inputs, _ = gen_random_inputs(inputs_descriptor, seed=seed)
                 input_batch = generate_input_batch(
@@ -754,7 +764,7 @@ def validate_network(network_onnx_path, raptor_path, onnx_include_dir,
 
                 if not reference_ready:
                     raise FileNotFoundError("reference runner is unavailable")
-                print_stage(reporter, model_index, model_total, network_onnx_path.name, "Run Reference")
+                print_stage(reporter, model_index, model_total, network_onnx_path.name, "Run reference")
                 reference_dirs = []
                 for index, flags in enumerate(input_flags):
                     reference_dir = workspace_dir / "outputs" / f"{index:06d}"