nnicolosi/Raptor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add validation script to validate all operations

Browse Source 
minor fixes
This commit is contained in:
NiccoloN
2026-02-26 16:11:22 +01:00
parent 33760e36ef
commit a42ff74a3b
4 changed files with 197 additions and 120 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
validation/raptor.py
View File

@@ -3,7 +3,7 @@ from pathlib import Path
from colorama import Fore, Style
def compile_with_raptor(network_path, raptor_onnx_path: Path, crossbar_size=64, crossbar_count=16):
def compile_with_raptor(network_path, raptor_onnx_path: Path, crossbar_size, crossbar_count):
# Define the arguments, with the possibility to set crossbar size and count
args = [
network_path,

175
validation/validate.py
View File

@@ -1,135 +1,76 @@
#!/usr/bin/env python3
import argparse
import json
import numpy as np
import subprocess
import sys
from pathlib import Path
from colorama import Style, Fore
from onnx_utils import gen_random_inputs, save_inputs_to_files, onnx_io, write_inputs_to_memory_bin, _ONNX_TO_NP
from raptor import compile_with_raptor
from gen_network_runner import gen_network_runner
from validate_one import validate_network
def compile_onnx_network(network_onnx_path, raptor_path, raptor_dir, runner_dir):
subprocess.run([raptor_path, network_onnx_path, "--EmitONNXIR"], check=True)
subprocess.run([raptor_path, network_onnx_path], check=True)
parent = network_onnx_path.parent
stem = network_onnx_path.stem
so_path = parent / f"{stem}.so"
mlir_path = parent / f"{stem}.onnx.mlir"
tmp_path = parent / f"{stem}.tmp"
moved_so = runner_dir / so_path.name
moved_mlir = raptor_dir / mlir_path.name
so_path.rename(moved_so)
mlir_path.rename(moved_mlir)
tmp_path.unlink(missing_ok=True)
return moved_so, moved_mlir
def discover_onnx_files(operations_dir):
return sorted(operations_dir.rglob("*.onnx"))
def build_onnx_runner(source_dir, build_dir):
subprocess.run(["cmake", source_dir], cwd=build_dir, check=True)
subprocess.run(["cmake", "--build", ".", "-j"], cwd=build_dir, check=True)
return build_dir / "runner"
def build_dump_ranges(config_path, outputs_descriptor):
with open(config_path) as f:
output_addresses = json.load(f)["outputs_addresses"]
ranges = []
for addr, (_, _, dtype_code, shape) in zip(output_addresses, outputs_descriptor):
byte_size = int(np.prod(shape)) * np.dtype(_ONNX_TO_NP[dtype_code]).itemsize
ranges.append(f"{addr},{byte_size}")
return ",".join(ranges)
def run_pim_simulator(simulator_dir, pim_dir, output_bin_path, dump_ranges):
subprocess.run(
["cargo", "run", "--release", "--package", "pim-simulator", "--bin", "pim-simulator", "--",
"-f", str(pim_dir), "-o", str(output_bin_path), "-d", dump_ranges],
cwd=simulator_dir, check=True
)
def parse_pim_simulator_outputs(output_bin_path, outputs_descriptor):
raw = output_bin_path.read_bytes()
arrays = []
offset = 0
for _, _, dtype_code, shape in outputs_descriptor:
dtype = np.dtype(_ONNX_TO_NP[dtype_code])
count = int(np.prod(shape))
array = np.frombuffer(raw, dtype=dtype, count=count, offset=offset).reshape(shape)
offset += count * dtype.itemsize
arrays.append(array)
return arrays
def validate_outputs(sim_arrays, runner_out_dir, outputs_descriptor, threshold=1e-3):
all_passed = True
for sim_array, (oi, name, _, shape) in zip(sim_arrays, outputs_descriptor):
csv_name = f"output{oi}_{name}.csv"
runner_array = np.loadtxt(runner_out_dir / csv_name, delimiter=',', dtype=np.float32).reshape(shape)
max_diff = float(np.max(np.abs(sim_array.astype(np.float64) - runner_array.astype(np.float64))))
passed = max_diff <= threshold
status = Fore.GREEN + "[PASS]" if passed else Fore.RED + "[FAIL]"
print(f" {name}: max diff = {max_diff:.6e} {status}" + Style.RESET_ALL)
if not passed:
all_passed = False
return all_passed
if __name__ == '__main__':
ap = argparse.ArgumentParser()
ap.add_argument("--network-onnx", required=True)
ap.add_argument("--raptor-path", required=True)
ap.add_argument("--onnx-include-dir", required=True)
def main():
ap = argparse.ArgumentParser(description="Validate all ONNX operations under the operations/ directory.")
ap.add_argument("--raptor-path", required=True, help="Path to the Raptor compiler binary.")
ap.add_argument("--onnx-include-dir", required=True, help="Path to OnnxMlirRuntime include directory.")
ap.add_argument("--operations-dir", default=None, help="Root of the operations tree (default: operations).")
ap.add_argument("--simulator-dir", default=None,
help="Path to pim-simulator crate root (default: auto-detected relative to script).")
ap.add_argument("--threshold", type=float, default=1e-3, help="Max allowed diff per output element.")
ap.add_argument("--crossbar-size", type=int, default=64)
ap.add_argument("--crossbar-count", type=int, default=8)
a = ap.parse_args()
network_onnx_path = Path(a.network_onnx).absolute()
raptor_path = a.raptor_path
script_dir = Path(__file__).parent.resolve()
operations_dir = Path(a.operations_dir) if a.operations_dir else script_dir / "operations"
simulator_dir = Path(a.simulator_dir) if a.simulator_dir else (
script_dir / ".." / "backend-simulators" / "pim" / "pim-simulator"
)
workspace_dir = network_onnx_path.parent
raptor_dir = workspace_dir / "raptor"
runner_dir = workspace_dir / "runner"
runner_build_dir = runner_dir / "build"
Path.mkdir(raptor_dir, exist_ok=True)
Path.mkdir(runner_build_dir, parents=True, exist_ok=True)
if not operations_dir.is_dir():
print(Fore.RED + f"Operations directory not found: {operations_dir}" + Style.RESET_ALL)
sys.exit(1)
print(Style.BRIGHT + "\nCompiling the onnx network:" + Style.RESET_ALL)
network_so_path, network_mlir_path = compile_onnx_network(network_onnx_path, raptor_path, raptor_dir, runner_dir)
onnx_files = discover_onnx_files(operations_dir)
if not onnx_files:
print(Fore.YELLOW + f"No .onnx files found under {operations_dir}" + Style.RESET_ALL)
sys.exit(1)
print(Style.BRIGHT + "\nGenerating and building the runner:" + Style.RESET_ALL)
gen_network_runner(network_onnx_path, network_so_path, a.onnx_include_dir, out=runner_dir / "runner.c")
runner_path = build_onnx_runner(runner_dir, runner_build_dir)
print(Style.BRIGHT + f"Found {len(onnx_files)} ONNX file(s) to validate.\n" + Style.RESET_ALL)
print(Style.BRIGHT + "\nGenerating random inputs:" + Style.RESET_ALL)
inputs_descriptor, outputs_descriptor = onnx_io(network_onnx_path)
inputs_list, inputs_dict = gen_random_inputs(inputs_descriptor)
flags, _files = save_inputs_to_files(network_onnx_path, inputs_list, out_dir=workspace_dir / "inputs")
results = {} # relative_path -> passed
for onnx_path in onnx_files:
rel = onnx_path.relative_to(operations_dir)
header = f"{'=' * 60}\n Validating: {rel}\n{'=' * 60}"
print(Style.BRIGHT + Fore.CYAN + header + Style.RESET_ALL)
print(Style.BRIGHT + "\nRunning inference with the runner:" + Style.RESET_ALL)
out_dir = workspace_dir / "outputs"
Path.mkdir(out_dir, exist_ok=True)
run_cmd = [runner_path, *flags]
run_cmd += ["--save-csv-dir", f"{out_dir}"]
subprocess.run(run_cmd, cwd=runner_build_dir, check=True)
try:
passed = validate_network(
onnx_path, a.raptor_path, a.onnx_include_dir, simulator_dir,
crossbar_size=a.crossbar_size, crossbar_count=a.crossbar_count,
threshold=a.threshold,
)
except Exception as e:
print(Fore.RED + f" ERROR: {e}" + Style.RESET_ALL)
passed = False
# Configuration parameters.
matrix_height = matrix_width = 512
crossbar_size = 64
crossbar_count = 8
results[str(rel)] = passed
print(Style.BRIGHT + "Compiling for PIM with Raptor:" + Style.RESET_ALL)
compile_with_raptor(network_mlir_path, raptor_path, crossbar_size, crossbar_count)
# Summary
n_passed = sum(results.values())
n_total = len(results)
print("\n" + Style.BRIGHT + "=" * 60)
print(" Summary")
print("=" * 60 + Style.RESET_ALL)
for rel, passed in results.items():
status = Fore.GREEN + "PASS" if passed else Fore.RED + "FAIL"
print(f" {rel}: {status}" + Style.RESET_ALL)
print(Style.BRIGHT + f"\n {n_passed}/{n_total} passed." + Style.RESET_ALL)
print(Style.BRIGHT + "\nRunning PIM simulation:" + Style.RESET_ALL)
pim_dir = raptor_dir / "pim"
write_inputs_to_memory_bin(pim_dir / "memory.bin", pim_dir / "config.json", inputs_list)
simulator_dir = Path(__file__).parent.resolve() / ".." / "backend-simulators" / "pim" / "pim-simulator"
simulation_dir = workspace_dir / "simulation"
Path.mkdir(simulation_dir, exist_ok=True)
dump_ranges = build_dump_ranges(pim_dir / "config.json", outputs_descriptor)
output_bin_path = simulation_dir / "out.bin"
run_pim_simulator(simulator_dir, pim_dir, output_bin_path, dump_ranges)
sys.exit(0 if n_passed == n_total else 1)
print(Style.BRIGHT + "\nValidating the results:" + Style.RESET_ALL)
sim_arrays = parse_pim_simulator_outputs(output_bin_path, outputs_descriptor)
validate_outputs(sim_arrays, out_dir, outputs_descriptor)
if __name__ == "__main__":
main()

137
validation/validate_one.py Normal file
View File

@@ -0,0 +1,137 @@
import argparse
import json
import numpy as np
import subprocess
from pathlib import Path
from colorama import Style, Fore
from onnx_utils import gen_random_inputs, save_inputs_to_files, onnx_io, write_inputs_to_memory_bin, _ONNX_TO_NP
from raptor import compile_with_raptor
from gen_network_runner import gen_network_runner
def compile_onnx_network(network_onnx_path, raptor_path, raptor_dir, runner_dir):
subprocess.run([raptor_path, network_onnx_path, "--EmitONNXIR"], check=True)
subprocess.run([raptor_path, network_onnx_path], check=True)
parent = network_onnx_path.parent
stem = network_onnx_path.stem
so_path = parent / f"{stem}.so"
mlir_path = parent / f"{stem}.onnx.mlir"
tmp_path = parent / f"{stem}.tmp"
moved_so = runner_dir / so_path.name
moved_mlir = raptor_dir / mlir_path.name
so_path.rename(moved_so)
mlir_path.rename(moved_mlir)
tmp_path.unlink(missing_ok=True)
return moved_so, moved_mlir
def build_onnx_runner(source_dir, build_dir):
subprocess.run(["cmake", source_dir], cwd=build_dir, check=True)
subprocess.run(["cmake", "--build", ".", "-j"], cwd=build_dir, check=True)
return build_dir / "runner"
def build_dump_ranges(config_path, outputs_descriptor):
with open(config_path) as f:
output_addresses = json.load(f)["outputs_addresses"]
ranges = []
for addr, (_, _, dtype_code, shape) in zip(output_addresses, outputs_descriptor):
byte_size = int(np.prod(shape)) * np.dtype(_ONNX_TO_NP[dtype_code]).itemsize
ranges.append(f"{addr},{byte_size}")
return ",".join(ranges)
def run_pim_simulator(simulator_dir, pim_dir, output_bin_path, dump_ranges):
subprocess.run(
["cargo", "run", "--release", "--package", "pim-simulator", "--bin", "pim-simulator", "--",
"-f", str(pim_dir), "-o", str(output_bin_path), "-d", dump_ranges],
cwd=simulator_dir, check=True
)
def parse_pim_simulator_outputs(output_bin_path, outputs_descriptor):
raw = output_bin_path.read_bytes()
arrays = []
offset = 0
for _, _, dtype_code, shape in outputs_descriptor:
dtype = np.dtype(_ONNX_TO_NP[dtype_code])
count = int(np.prod(shape))
array = np.frombuffer(raw, dtype=dtype, count=count, offset=offset).reshape(shape)
offset += count * dtype.itemsize
arrays.append(array)
return arrays
def validate_outputs(sim_arrays, runner_out_dir, outputs_descriptor, threshold=1e-3):
all_passed = True
for sim_array, (oi, name, _, shape) in zip(sim_arrays, outputs_descriptor):
csv_name = f"output{oi}_{name}.csv"
runner_array = np.loadtxt(runner_out_dir / csv_name, delimiter=',', dtype=np.float32).reshape(shape)
max_diff = float(np.max(np.abs(sim_array.astype(np.float64) - runner_array.astype(np.float64))))
passed = max_diff <= threshold
status = Fore.GREEN + "[PASS]" if passed else Fore.RED + "[FAIL]"
print(f" {name}: max diff = {max_diff:.6e} {status}" + Style.RESET_ALL)
if not passed:
all_passed = False
return all_passed
def validate_network(network_onnx_path, raptor_path, onnx_include_dir,
simulator_dir, crossbar_size=64, crossbar_count=8, threshold=1e-3):
network_onnx_path = Path(network_onnx_path).absolute()
workspace_dir = network_onnx_path.parent
raptor_dir = workspace_dir / "raptor"
runner_dir = workspace_dir / "runner"
runner_build_dir = runner_dir / "build"
Path.mkdir(raptor_dir, exist_ok=True)
Path.mkdir(runner_build_dir, parents=True, exist_ok=True)
print(Style.BRIGHT + "\nCompiling the onnx network:" + Style.RESET_ALL)
network_so_path, network_mlir_path = compile_onnx_network(network_onnx_path, raptor_path, raptor_dir, runner_dir)
print(Style.BRIGHT + "\nGenerating and building the runner:" + Style.RESET_ALL)
gen_network_runner(network_onnx_path, network_so_path, onnx_include_dir, out=runner_dir / "runner.c")
runner_path = build_onnx_runner(runner_dir, runner_build_dir)
print(Style.BRIGHT + "\nGenerating random inputs:" + Style.RESET_ALL)
inputs_descriptor, outputs_descriptor = onnx_io(network_onnx_path)
inputs_list, inputs_dict = gen_random_inputs(inputs_descriptor)
flags, _files = save_inputs_to_files(network_onnx_path, inputs_list, out_dir=workspace_dir / "inputs")
print(Style.BRIGHT + "\nRunning inference with the runner:" + Style.RESET_ALL)
out_dir = workspace_dir / "outputs"
Path.mkdir(out_dir, exist_ok=True)
run_cmd = [runner_path, *flags]
run_cmd += ["--save-csv-dir", f"{out_dir}"]
subprocess.run(run_cmd, cwd=runner_build_dir, check=True)
print(Style.BRIGHT + "\nCompiling for PIM with Raptor:" + Style.RESET_ALL)
compile_with_raptor(network_mlir_path, raptor_path, crossbar_size, crossbar_count)
print(Style.BRIGHT + "\nRunning PIM simulation:" + Style.RESET_ALL)
pim_dir = raptor_dir / "pim"
write_inputs_to_memory_bin(pim_dir / "memory.bin", pim_dir / "config.json", inputs_list)
simulation_dir = workspace_dir / "simulation"
Path.mkdir(simulation_dir, exist_ok=True)
dump_ranges = build_dump_ranges(pim_dir / "config.json", outputs_descriptor)
output_bin_path = simulation_dir / "out.bin"
run_pim_simulator(simulator_dir, pim_dir, output_bin_path, dump_ranges)
print(Style.BRIGHT + "\nValidating the results:" + Style.RESET_ALL)
sim_arrays = parse_pim_simulator_outputs(output_bin_path, outputs_descriptor)
return validate_outputs(sim_arrays, out_dir, outputs_descriptor, threshold)
if __name__ == '__main__':
ap = argparse.ArgumentParser()
ap.add_argument("--network-onnx", required=True)
ap.add_argument("--raptor-path", required=True)
ap.add_argument("--onnx-include-dir", required=True)
a = ap.parse_args()
simulator_dir = Path(__file__).parent.resolve() / ".." / "backend-simulators" / "pim" / "pim-simulator"
passed = validate_network(
a.network_onnx, a.raptor_path, a.onnx_include_dir, simulator_dir
)
raise SystemExit(0 if passed else 1)