add throughput mode to validation scripts

make raptor also emit input sizes
This commit is contained in:
NiccoloN
2026-08-11 10:34:50 +02:00
parent 910701dfaf
commit c55d9f3dad
15 changed files with 974 additions and 497 deletions
@@ -15,7 +15,7 @@ REPO_ROOT = VALIDATION_DIR.parent
if str(VALIDATION_DIR) not in sys.path:
sys.path.insert(0, str(VALIDATION_DIR))
from raptor_validation.onnx_utils import _ONNX_TO_NP, onnx_io, write_inputs_to_memory_bin
from raptor_validation.onnx_utils import _ONNX_TO_NP, onnx_io, write_inputs_binary, write_inputs_to_memory_bin
from raptor_validation.validate_one import (
MODE_COMPILE_ONLY,
build_dump_ranges,
@@ -222,6 +222,8 @@ def run_reference_and_simulator(args, model_path: Path, tensor: np.ndarray):
subprocess.run(runner_cmd, cwd=runner_build_dir, check=True)
write_inputs_to_memory_bin(pim_dir / "memory.bin", pim_dir / "config.json", [tensor])
input_bin_path = simulation_dir / "input.bin"
write_inputs_binary(input_bin_path, [tensor])
dump_ranges = build_dump_ranges(pim_dir / "config.json", output_descriptors)
output_bin_path = simulation_dir / "out.bin"
run_pim_simulator(
@@ -230,6 +232,7 @@ def run_reference_and_simulator(args, model_path: Path, tensor: np.ndarray):
output_bin_path,
dump_ranges,
timeout_sec=args.command_timeout_seconds,
input_paths=[input_bin_path],
)
output_index, output_name, output_dtype_code, output_shape = output_descriptors[0]
@@ -33,18 +33,22 @@ sys.path.insert(0, str(VALIDATION_DIR))
from raptor_validation.gen_network_runner import gen_network_runner # noqa: E402
from raptor_validation.onnx_utils import ( # noqa: E402
_ONNX_TO_NP,
generate_input_batch,
gen_random_inputs,
onnx_io,
save_inputs_to_files,
write_input_batch_binaries,
write_input_batch_csv,
write_inputs_to_memory_bin,
)
from raptor_validation.raptor import compile_with_raptor # noqa: E402
from raptor_validation.pimsim_nn import ( # noqa: E402
export_raptor_latency_artifact,
export_raptor_pimsim_artifact,
parse_pimsim_nn_metrics,
)
from raptor_validation.validate_one import ( # noqa: E402
STAGE_COLORS,
build_pim_simulator_command,
build_dump_ranges,
parse_pim_simulator_outputs,
)
@@ -402,6 +406,29 @@ def generate_reference_outputs(
return reference_dir
def generate_reference_batch_outputs(
runner_path: Path,
runner_build_dir: Path,
model_path: Path,
input_batch: list[list[np.ndarray]],
steps: list[StepRecord],
args: argparse.Namespace,
out_dir: Path,
) -> list[Path]:
return [
generate_reference_outputs(
runner_path,
runner_build_dir,
model_path,
sample,
steps,
args,
out_dir / f"batch_{index:06d}",
)
for index, sample in enumerate(input_batch)
]
def prepare_common_artifacts(
args: argparse.Namespace,
model_path: Path,
@@ -500,31 +527,31 @@ def run_functional_validation(
pim_dir: Path,
config_path: Path,
output_bin: Path,
input_bins: list[Path],
outputs_desc: list[tuple[int, str, int, list[int]]],
reference_dir: Path,
reference_dirs: list[Path],
steps: list[StepRecord],
args: argparse.Namespace,
*,
channel_last: bool = False,
) -> CompareResult:
dump_ranges = build_dump_ranges(config_path, outputs_desc)
cmd = [
"cargo",
"run",
"--no-default-features",
"--release",
"--package",
"pim-simulator",
"--bin",
"pim-simulator",
"--",
"-f",
str(pim_dir),
"-o",
str(output_bin),
"-d",
batch_size = len(input_bins)
if batch_size == 0 or len(reference_dirs) != batch_size:
raise ValueError(
f"functional validation requires one input and reference per iteration, got "
f"{batch_size} inputs and {len(reference_dirs)} references"
)
batch_output_dir = output_bin.parent / f"{output_bin.stem}_iterations"
shutil.rmtree(batch_output_dir, ignore_errors=True)
cmd = build_pim_simulator_command(
pim_dir,
output_bin,
dump_ranges,
]
input_bins,
args.pimsim_mode,
batch_output_dir,
)
output_bin.parent.mkdir(parents=True, exist_ok=True)
run_logged(
label,
@@ -534,13 +561,25 @@ def run_functional_validation(
steps=steps,
stage="Run Functional Simulation",
)
return compare_simulator_outputs(
output_bin,
outputs_desc,
reference_dir,
threshold=args.threshold,
rtol=args.rtol,
channel_last=channel_last,
max_diffs: dict[str, float] = {}
failed_iterations = []
for index, reference_dir in enumerate(reference_dirs):
result = compare_simulator_outputs(
batch_output_dir / f"output_{index:06d}.bin",
outputs_desc,
reference_dir,
threshold=args.threshold,
rtol=args.rtol,
channel_last=channel_last,
)
if not result.passed:
failed_iterations.append(index)
for name, diff in result.max_diffs.items():
max_diffs[name] = max(max_diffs.get(name, 0.0), diff)
return CompareResult(
passed=not failed_iterations,
max_diffs=max_diffs,
error=(f"batch iterations failed: {failed_iterations}" if failed_iterations else None),
)
@@ -730,6 +769,7 @@ def export_pimcomp_for_rust(
"adc_count": sim_info["config"]["adc_count"],
"array_group_map": {},
"inputs_addresses": [input_addr],
"inputs_sizes": [input_tensor.nbytes],
"outputs_addresses": [],
}
output_name_to_node = {node["name"]: node for node in node_list}
@@ -887,6 +927,8 @@ def run_pimsim_nn(
str(config_path),
"--gui=false",
]
if not args.no_fast:
cmd.append("--fast")
output = run_logged(
label,
cmd,
@@ -997,8 +1039,12 @@ def perf_status(perf: dict[str, Any]) -> str:
return "DONE"
def perf_value(perf: dict[str, Any], key: str) -> Any:
return perf[key] if key in perf else "n/a"
def perf_value(perf: dict[str, Any], key: str, unit: str = "") -> Any:
value = perf.get(key)
if value is None:
return "n/a"
formatted = f"{value:.2f}" if isinstance(value, float) else value
return f"{formatted} {unit}" if unit else formatted
def empty_instruction_summary(reason: str | None = None, error: str | None = None) -> dict[str, Any]:
@@ -1170,24 +1216,24 @@ def write_report(
if pimsim_mode == "throughput":
lines.extend(
[
"| Compiler | Status | Throughput (samples/s) | Avg latency (ms) | Avg power (mW) | Avg energy (pJ/it) | Output count |",
"| Compiler | Status | Avg latency | Throughput | Avg power | Avg energy | Output count |",
"|---|---|---:|---:|---:|---:|---:|",
f"| Raptor | {perf_status(raptor_perf)} | {perf_value(raptor_perf, 'throughput')} | {perf_value(raptor_perf, 'average_latency_ms')} | "
f"{perf_value(raptor_perf, 'average_power_mw')} | {perf_value(raptor_perf, 'average_energy_pj')} | {perf_value(raptor_perf, 'output_count')} |",
f"| PIMCOMP | {perf_status(pimcomp_perf)} | {perf_value(pimcomp_perf, 'throughput')} | {perf_value(pimcomp_perf, 'average_latency_ms')} | "
f"{perf_value(pimcomp_perf, 'average_power_mw')} | {perf_value(pimcomp_perf, 'average_energy_pj')} | {perf_value(pimcomp_perf, 'output_count')} |",
f"| Raptor | {perf_status(raptor_perf)} | {perf_value(raptor_perf, 'average_latency_ms', 'ms')} | {perf_value(raptor_perf, 'throughput', 'samples/s')} | "
f"{perf_value(raptor_perf, 'average_power_mw', 'mW')} | {perf_value(raptor_perf, 'average_energy_pj', 'pJ/it')} | {perf_value(raptor_perf, 'output_count')} |",
f"| PIMCOMP | {perf_status(pimcomp_perf)} | {perf_value(pimcomp_perf, 'average_latency_ms', 'ms')} | {perf_value(pimcomp_perf, 'throughput', 'samples/s')} | "
f"{perf_value(pimcomp_perf, 'average_power_mw', 'mW')} | {perf_value(pimcomp_perf, 'average_energy_pj', 'pJ/it')} | {perf_value(pimcomp_perf, 'output_count')} |",
"",
]
)
else:
lines.extend(
[
"| Compiler | Status | Latency (ms) | Avg power (mW) | Avg energy (pJ) |",
"| Compiler | Status | Latency | Avg power | Avg energy |",
"|---|---|---:|---:|---:|",
f"| Raptor | {perf_status(raptor_perf)} | {perf_value(raptor_perf, 'latency_ms')} | "
f"{perf_value(raptor_perf, 'average_power_mw')} | {perf_value(raptor_perf, 'average_energy_pj')} |",
f"| PIMCOMP | {perf_status(pimcomp_perf)} | {perf_value(pimcomp_perf, 'latency_ms')} | "
f"{perf_value(pimcomp_perf, 'average_power_mw')} | {perf_value(pimcomp_perf, 'average_energy_pj')} |",
f"| Raptor | {perf_status(raptor_perf)} | {perf_value(raptor_perf, 'latency_ms', 'ms')} | "
f"{perf_value(raptor_perf, 'average_power_mw', 'mW')} | {perf_value(raptor_perf, 'average_energy_pj', 'pJ')} |",
f"| PIMCOMP | {perf_status(pimcomp_perf)} | {perf_value(pimcomp_perf, 'latency_ms', 'ms')} | "
f"{perf_value(pimcomp_perf, 'average_power_mw', 'mW')} | {perf_value(pimcomp_perf, 'average_energy_pj', 'pJ')} |",
"",
]
)
@@ -1293,6 +1339,7 @@ def main():
parser.add_argument("--mesh-cols", type=int)
parser.add_argument("--pimsim-time-ms", type=int, default=1000)
parser.add_argument("--pimsim-mode", choices=["latency", "throughput"], default="latency")
parser.add_argument("--batch-size", type=int, default=128)
parser.add_argument("--pimcomp-pipeline", choices=["element", "batch"])
parser.add_argument("--pimcomp-model-name", help="Use a PIMCOMP built-in model name such as vgg16.")
parser.add_argument(
@@ -1316,6 +1363,11 @@ def main():
help="Preserve the PIMCOMP side of an existing comparison report without rerunning it.",
)
parser.add_argument("--skip-pimsim-nn", action="store_true")
parser.add_argument(
"--no-fast",
action="store_true",
help="Disable fast pimsim-nn throughput convergence for authoritative experiments.",
)
parser.add_argument("--verbose-raptor-compile", action="store_true")
parser.add_argument("--raptor-extra-arg", action="append", default=[])
parser.add_argument(
@@ -1328,6 +1380,10 @@ def main():
parser.error("--reuse-pimcomp-dir and --reuse-pimcomp-report are mutually exclusive")
if args.pimsim_time_ms <= 0:
parser.error("--pimsim-time-ms must be positive")
if args.batch_size <= 0:
parser.error("--batch-size must be positive")
if args.pimsim_mode == "throughput" and args.batch_size < 2:
parser.error("throughput mode requires batch size greater than 1")
if args.timeout_seconds < 0:
parser.error("--timeout-seconds must be non-negative")
if args.pimcomp_pipeline is None:
@@ -1371,6 +1427,7 @@ def main():
runner_path: Path | None = None
reference_dir: Path | None = None
reference_dirs: list[Path] = []
raptor_pim_dir: Path | None = None
raptor_pimsim_dir: Path | None = None
raptor_pass_timings: dict[str, float] = {}
@@ -1506,6 +1563,33 @@ def main():
"Reference outputs were skipped because the native runner or model inputs are not available.",
)
input_batch = None
raptor_input_bins: list[Path] = []
pimcomp_input_bins: list[Path] = []
if model_io is not None:
batch_size = 1 if args.pimsim_mode == "latency" else args.batch_size
input_batch = generate_input_batch(inputs_desc, runtime_inputs, batch_size, args.seed)
write_input_batch_csv(out_dir / "inputs.csv", input_batch)
raptor_input_bins = write_input_batch_binaries(input_batch, out_dir / "simulation/raptor_inputs")
if args.pimsim_mode == "throughput":
throughput_references = try_stage(
failures,
"Run throughput references",
generate_reference_batch_outputs,
runner_path,
runner_path.parent,
model_path,
input_batch,
steps,
args,
out_dir / "reference",
) if runner_path is not None and runner_path.exists() else None
if throughput_references is not None:
reference_dirs = throughput_references
reference_dir = out_dir / "reference"
elif reference_dir is not None:
reference_dirs = [reference_dir]
if not reuse_raptor and model_path.exists() and hardware["core_count"] > 0:
compiled_raptor = try_stage(
failures,
@@ -1535,7 +1619,7 @@ def main():
raptor_pim_dir / "config.json",
runtime_inputs,
)
if wrote_inputs and reference_dir is not None and outputs_desc:
if wrote_inputs and reference_dirs and outputs_desc:
validation = try_stage(
failures,
"Functional Validation Raptor",
@@ -1544,13 +1628,14 @@ def main():
raptor_pim_dir,
raptor_pim_dir / "config.json",
out_dir / "simulation/out.bin",
raptor_input_bins,
outputs_desc,
reference_dir,
reference_dirs,
steps,
args,
)
raptor_validation = validation if validation is not None else failed_validation("Raptor validation failed")
elif reference_dir is None:
elif not reference_dirs:
raptor_validation = skipped_validation("Reference outputs are not available")
elif not outputs_desc:
raptor_validation = skipped_validation("Output descriptors are not available")
@@ -1616,7 +1701,14 @@ def main():
"PIMCOMP functional export failed because model inputs are not available.",
)
if not reuse_pimcomp and pimcomp_export_dir is not None and reference_dir is not None and outputs_desc:
if input_batch is not None and pimcomp_export_dir is not None:
pimcomp_input_bins = write_input_batch_binaries(
input_batch,
out_dir / "simulation/pimcomp_inputs",
transform=flatten_pimcomp_input,
)
if not reuse_pimcomp and pimcomp_export_dir is not None and reference_dirs and outputs_desc:
validation = try_stage(
failures,
"Functional Validation PIMCOMP",
@@ -1625,8 +1717,9 @@ def main():
pimcomp_export_dir,
pimcomp_export_dir / "config.json",
out_dir / "simulation/pimcomp.out.bin",
pimcomp_input_bins,
outputs_desc,
reference_dir,
reference_dirs,
steps,
args,
channel_last=True,
@@ -1636,7 +1729,7 @@ def main():
pass
elif pimcomp_export_dir is None:
pimcomp_validation = failed_validation("PIMCOMP functional export is not available")
elif reference_dir is None:
elif not reference_dirs:
pimcomp_validation = failed_validation("Reference outputs are not available")
else:
pimcomp_validation = failed_validation("Output descriptors are not available")
@@ -1673,7 +1766,7 @@ def main():
raptor_pimsim_dir = try_stage(
failures,
"Export Raptor for pimsim-nn",
export_raptor_latency_artifact,
export_raptor_pimsim_artifact,
raptor_pim_dir,
out_dir / "raptor/pimsim_nn",
)
@@ -1777,6 +1870,8 @@ def main():
"common_dir": str(common_dir),
"reference_inputs": optional_path(common_dir / "inputs"),
"reference_outputs": optional_path(reference_dir),
"batch_inputs": optional_path(out_dir / "inputs.csv"),
"batch_outputs": optional_path(out_dir / "simulation/out_iterations"),
"reference_runner": optional_path(runner_path),
"raptor_pim": optional_path(raptor_pim_dir),
"raptor_pimsim_nn": optional_path(raptor_pimsim_dir),
@@ -108,10 +108,10 @@ def write_results_csv(
"pimcomp_pipeline",
"raptor_functional_validation",
"pimcomp_functional_validation",
"raptor_throughput_samples_s",
"pimcomp_throughput_samples_s",
"raptor_latency_ms",
"pimcomp_latency_ms",
"raptor_throughput_samples_s",
"pimcomp_throughput_samples_s",
"raptor_power_mw",
"pimcomp_power_mw",
"raptor_energy_pj",
@@ -215,13 +215,13 @@ def write_results_csv(
def performance_values(performance: dict) -> dict[str, float | None]:
parsed = parse_pimsim_nn_metrics(performance.get("raw_output", ""))
return {
"throughput": performance.get("throughput") or parsed.get("throughput"),
"latency": (
performance.get("latency_ms")
or performance.get("average_latency_ms")
or parsed.get("latency_ms")
or parsed.get("average_latency_ms")
),
"throughput": performance.get("throughput") or parsed.get("throughput"),
"power": performance.get("average_power_mw") or parsed.get("average_power_mw"),
"energy": performance.get("average_energy_pj") or parsed.get("average_energy_pj"),
}
@@ -261,7 +261,7 @@ def comparison_winner(mode: str, raptor: float, pimcomp: float) -> str:
def format_value(value: float | None) -> str:
return "NA" if value is None else f"{value:.6f}"
return "NA" if value is None else f"{value:.2f}"
def print_stage(title: str, color: str) -> None:
@@ -323,7 +323,9 @@ def comparison_command(
pipeline: int,
pimcomp_pipeline: str,
pimsim_time_ms: int,
batch_size: int,
timeout: float,
fast: bool,
reuse_raptor_report: Path | None = None,
reuse_pimcomp_dir: Path | None = None,
reuse_pimcomp_report: Path | None = None,
@@ -351,6 +353,8 @@ def comparison_command(
str(config),
"--pimsim-mode",
mode,
"--batch-size",
str(batch_size),
*time_args,
"--pimcomp-pipeline",
pimcomp_pipeline,
@@ -360,6 +364,7 @@ def comparison_command(
"--timeout-seconds",
str(timeout),
"--fail-on-error",
*([] if fast else ["--no-fast"]),
*reuse_args,
]
@@ -396,7 +401,9 @@ def comparison_command_for(
spec.pipeline,
spec.pimcomp_pipeline,
args.pimsim_time_ms,
args.batch_size,
args.timeout_seconds,
not args.no_fast,
reuse_raptor_report=(
report
if args.only == "pimcomp"
@@ -512,8 +519,14 @@ def main() -> int:
parser.add_argument(
"--pimsim-time-ms",
type=int,
default=100,
help="throughput pimsim-nn horizon in ms (default: 100).",
default=1000,
help="throughput pimsim-nn convergence deadline in ms (default: 1000).",
)
parser.add_argument(
"--batch-size",
type=int,
default=128,
help="functional throughput batch size (default: 128).",
)
parser.add_argument(
"--timeout-seconds",
@@ -525,8 +538,8 @@ def main() -> int:
"-j",
"--jobs",
type=int,
default=os.cpu_count() or 1,
help="Number of comparisons to run in parallel (default: all available CPUs).",
default=4,
help="Number of comparisons to run in parallel (default: 4).",
)
parser.add_argument(
"--clean",
@@ -534,6 +547,11 @@ def main() -> int:
help="Remove generated comparison artifacts and result summaries, then exit.",
)
parser.add_argument("--dry-run", action="store_true", help="Print commands without modifying files.")
parser.add_argument(
"--no-fast",
action="store_true",
help="Disable fast pimsim-nn throughput convergence for authoritative experiments.",
)
args = parser.parse_args()
out_dir = args.out_dir.resolve() if args.out_dir is not None else None
@@ -547,6 +565,8 @@ def main() -> int:
parser.error("--jobs must be at least 1")
if args.pimsim_time_ms <= 0:
parser.error("--pimsim-time-ms must be positive")
if args.batch_size <= 0:
parser.error("--batch-size must be positive")
if args.timeout_seconds < 0:
parser.error("--timeout-seconds must be non-negative")
comparisons_by_arch: dict[str, tuple[tuple[str, int, str], ...]] = {}
@@ -21,7 +21,7 @@ if sys.version_info < (3, 10):
"Run it with a newer interpreter, for example your project venv Python."
)
from raptor_validation.onnx_utils import _ONNX_TO_NP, onnx_io, write_inputs_to_memory_bin
from raptor_validation.onnx_utils import _ONNX_TO_NP, onnx_io, write_inputs_binary, write_inputs_to_memory_bin
from raptor_validation.validate_one import (
MODE_COMPILE_ONLY,
build_dump_ranges,
@@ -137,6 +137,8 @@ def run_local_reference_and_simulator(args, network_dir: Path, network_onnx_path
tensor = np.loadtxt(paths["input_csv"], delimiter=",", dtype=np.float32).reshape(1, 3, 640, 640)
write_inputs_to_memory_bin(paths["raptor_pim"] / "memory.bin", paths["raptor_pim"] / "config.json", [tensor])
input_bin = paths["sim_dir"] / "input.bin"
write_inputs_binary(input_bin, [tensor])
dump_ranges = build_dump_ranges(paths["raptor_pim"] / "config.json", output_descriptors)
run_pim_simulator(
@@ -145,6 +147,7 @@ def run_local_reference_and_simulator(args, network_dir: Path, network_onnx_path
paths["sim_bin"],
dump_ranges,
timeout_sec=args.command_timeout_seconds,
input_paths=[input_bin],
)
return paths, output_descriptors[0]
@@ -272,6 +272,7 @@ def remote_case_paths(args, case_name: str):
"input_csv": root / "real_image_validation" / "inputs" / f"{case_name}.csv",
"ref_dir": root / "real_image_validation" / "reference" / case_name,
"sim_dir": root / "real_image_validation" / "simulation" / case_name,
"sim_input": root / "real_image_validation" / "simulation" / case_name / "input.bin",
"sim_bin": root / "real_image_validation" / "simulation" / case_name / "out.bin",
}
@@ -310,8 +311,10 @@ import numpy as np
from pathlib import Path
input_csv = Path({json.dumps(str(paths["input_csv"]))})
pim_dir = Path({json.dumps(str(paths["raptor_pim"]))})
input_bin = Path({json.dumps(str(paths["sim_input"]))})
config = json.loads((pim_dir / "config.json").read_text())
tensor = np.loadtxt(input_csv, delimiter=",", dtype=np.float32).reshape(1, 3, 640, 640)
input_bin.write_bytes(tensor.tobytes(order="C"))
with open(pim_dir / "memory.bin", "r+b") as f:
f.seek(config["inputs_addresses"][0])
f.write(tensor.tobytes(order="C"))
@@ -327,7 +330,8 @@ PY
f"export PATH=$HOME/.cargo/bin:$PATH && "
f"cd {quoted_project}/backend-simulators/pim/pim-simulator && "
f"cargo run --no-default-features --release --package pim-simulator --bin pim-simulator -- "
f"-f {quoted_pim} -o {quoted_sim_bin} -d {dump_range}"
f"-f {quoted_pim} -o {quoted_sim_bin} -d {dump_range} "
f"--mode latency --input {shlex.quote(str(paths['sim_input']))}"
)
remote_bash(args.ssh_key, args.remote_host, sim_command)
return paths