From 9a7168e7b666afbfd90ee4dbcc3a14db19135693 Mon Sep 17 00:00:00 2001 From: NiccoloN Date: Wed, 25 Feb 2026 09:44:42 +0100 Subject: [PATCH] add automatic validation process wit pim simulator --- onnx-mlir | 2 +- validation/.gitignore | 5 + validation/gen_network_runner.py | 274 ++++++++++++++++++ validation/onnx_utils.py | 200 +++++++++++++ .../operations/gemm/simple/GemmTest5.onnx | Bin 0 -> 69804 bytes validation/raptor.py | 29 ++ validation/validate.py | 135 +++++++++ 7 files changed, 644 insertions(+), 1 deletion(-) create mode 100644 validation/.gitignore create mode 100644 validation/gen_network_runner.py create mode 100644 validation/onnx_utils.py create mode 100644 validation/operations/gemm/simple/GemmTest5.onnx create mode 100644 validation/raptor.py create mode 100644 validation/validate.py diff --git a/onnx-mlir b/onnx-mlir index 840d057..84cedd1 160000 --- a/onnx-mlir +++ b/onnx-mlir @@ -1 +1 @@ -Subproject commit 840d05752072bf9d0488c00c4389c43a7e5dc6df +Subproject commit 84cedd1d690d1c04056caec7ba29be1abea91815 diff --git a/validation/.gitignore b/validation/.gitignore new file mode 100644 index 0000000..70dfb95 --- /dev/null +++ b/validation/.gitignore @@ -0,0 +1,5 @@ +operations/**/inputs +operations/**/outputs +operations/**/raptor +operations/**/runner +operations/**/simulation diff --git a/validation/gen_network_runner.py b/validation/gen_network_runner.py new file mode 100644 index 0000000..99650cf --- /dev/null +++ b/validation/gen_network_runner.py @@ -0,0 +1,274 @@ +#!/usr/bin/env python3 + +import argparse, os, pathlib, textwrap +from onnx_utils import onnx_io +from onnx import TensorProto + +# ONNX dtype -> (ctype, printf, ONNX_TYPE_*) +DTYPES = { + TensorProto.FLOAT: ("float", "%g", "ONNX_TYPE_FLOAT"), + TensorProto.DOUBLE: ("double", "%g", "ONNX_TYPE_DOUBLE"), + TensorProto.INT64: ("int64_t", "%lld","ONNX_TYPE_INT64"), + TensorProto.INT32: ("int32_t", "%d", "ONNX_TYPE_INT32"), + TensorProto.UINT8: ("uint8_t", "%u", "ONNX_TYPE_UINT8"), + TensorProto.INT8: ("int8_t", "%d", "ONNX_TYPE_INT8"), + TensorProto.BOOL: ("uint8_t", "%u", "ONNX_TYPE_BOOL"), # stored as byte + TensorProto.FLOAT16: ("uint16_t", "%u", "ONNX_TYPE_FLOAT16"), # raw 16-bit + TensorProto.BFLOAT16:("uint16_t", "%u", "ONNX_TYPE_BFLOAT16"), +} + +def esc(s): return s.replace("\\","\\\\").replace('"','\\"') + +def gen_c(inputs, outputs, entry, so_name): + in_blocks=[] + for i,name,et,shape in inputs: + if et not in DTYPES: + raise ValueError(f"Unsupported dtype for input '{name}': {et}") + cty, pfmt, onnx_ty = DTYPES[et] + shp_list = ", ".join(str(d) for d in shape) if shape else "" + rank = len(shape) + in_blocks.append(textwrap.dedent(f""" + // ---- Input {i}: "{esc(name)}" ({cty}) ---- + const char *in{i}_csv=NULL, *in{i}_csv_file=NULL, *in{i}_shape_str=NULL; + char *in{i}_csv_buf=NULL; // holds file contents if --in{i}-csv-file used + int has_in{i}=0; double in{i}_fill=0.0; int in{i}_fill_set=0; + + for (int ai=1; ai=in{i}_nelem) break; + double v=atof(tok); + (({cty}*)in{i}_buf)[idx++] = ({cty})v; + tok=strtok(NULL,",\\n\\r\\t "); + }} + free(buf); + if(idx!=in{i}_nelem){{fprintf(stderr,"ERROR: CSV provided %lld values, expected %lld.\\n",(long long)idx,in{i}_nelem); if(in{i}_csv_buf) free(in{i}_csv_buf); return 2;}} + }} else if (in{i}_fill_set) {{ + {cty} vv=({cty})in{i}_fill; for(long long t=0;t.csv" + char fname[512]; + // simple sanitizer: copy name => replace non [A-Za-z0-9_.-] with '_' + char clean[256]; int ci=0; const char *src="{safe}"; + for (; src[ci] && ci < 255; ++ci) {{ + char ch = src[ci]; + int ok = (ch>='A'&&ch<='Z')||(ch>='a'&&ch<='z')||(ch>='0'&&ch<='9')||ch=='_'||ch=='-'||ch=='.'; + clean[ci] = ok ? ch : '_'; + }} + clean[ci] = '\\0'; + snprintf(fname, sizeof(fname), "%s/output{oi}_%s.csv", save_csv_dir, clean); + FILE *csv = fopen(fname, "w"); + if (!csv) {{ perror("fopen --save-csv-dir"); }} + else {{ + OMTensor *t = omTensorListGetOmtByIndex(out_list, {oi}); + int64_t rank = omTensorGetRank(t); + int64_t const *shape = omTensorGetShape(t); + long long numel = 1; for (int64_t k=0;k +#include +#include +#include + +OMTensorList *{entry}(OMTensorList *inputs); + +int main(int argc, char **argv) {{ + // optional: --save-csv-dir (directory must exist) + const char *save_csv_dir = NULL; + for (int ai=1; ai tuple/list of dims. + Overrides the shape inferred from the model (useful for dynamic dims). + float_range: + Range for floats (uniform). + int_range: + Range for integers (uniform integers, inclusive of low/high with np.integers semantics). + dyn_dim_default: + If a dim is dynamic/unknown, use this value (unless shape_overrides provides one). + seed: + RNG seed for reproducibility. + + Returns + ------- + inputs_list: list[np.ndarray] + Arrays in graph input order (index-sorted). + inputs_dict: dict[str, np.ndarray] + Mapping input_name -> array in the ONNX-declared dtype. + """ + rng = np.random.default_rng(seed) + ins = onnx_inputs + + # Normalize overrides to support both index and name keys. + shape_overrides = shape_overrides or {} + name_overrides = {k: tuple(v) for k, v in shape_overrides.items() if isinstance(k, str)} + idx_overrides = {int(k): tuple(v) for k, v in shape_overrides.items() if isinstance(k, int)} + + arrays_by_name = {} + arrays_in_order = [] + + for idx, name, elem_type, shape in ins: + # Resolve dtype + if elem_type not in _ONNX_TO_NP: + raise ValueError(f"Unsupported ONNX dtype for input '{name}': {elem_type}") + np_dtype = _ONNX_TO_NP[elem_type] + + # Resolve shape: model -> replace unknowns with dyn_dim_default -> apply overrides + resolved_shape = list(shape or []) + if not resolved_shape: + resolved_shape = [dyn_dim_default] # scalar-like: treat as 1-dim with size dyn_dim_default + + # If your onnx_io already sets unknown dims to 1, we still allow overriding: + if idx in idx_overrides: + resolved_shape = list(idx_overrides[idx]) + elif name in name_overrides: + resolved_shape = list(name_overrides[name]) + + # Make sure no zeros + resolved_shape = [int(d if d and d > 0 else dyn_dim_default) for d in resolved_shape] + size = int(np.prod(resolved_shape)) + + # Generate data + if np.issubdtype(np_dtype, np.floating): + lo, hi = float_range + # generate in float32/64 and cast as needed + base_dtype = np.float32 if np_dtype in (np.float16, getattr(np, "bfloat16", np.float32)) else np_dtype + arr = rng.uniform(lo, hi, size=size).astype(base_dtype).reshape(resolved_shape) + # cast to f16/bf16 if required + if np_dtype is np.float16: + arr = arr.astype(np.float16) + elif getattr(np, "bfloat16", None) is not None and np_dtype is np.bfloat16: + arr = arr.astype(np.bfloat16) + elif np_dtype == np.uint8 and elem_type == TensorProto.BOOL: + # Bool as 0/1 bytes + arr = (rng.random(size=size) < 0.5).astype(np.uint8).reshape(resolved_shape) + elif np.issubdtype(np_dtype, np.integer): + lo, hi = int_range + bounds = _dtype_bounds(np_dtype) + if bounds is not None: + lo = max(lo, bounds[0]) + hi = min(hi, bounds[1]) + # np.random.integers is exclusive of high; add 1 for int range + arr = rng.integers(lo, hi + 1, size=size, dtype=np_dtype).reshape(resolved_shape) + else: + raise ValueError(f"Unhandled dtype mapping for input '{name}' (elem_type={elem_type}).") + + arrays_by_name[name] = arr + arrays_in_order.append(arr) + return arrays_in_order, arrays_by_name + + +def save_inputs_to_files(onnx_path, arrays_in_order, out_dir): + """ + Save arrays to CSV files. Returns (flags, files) where flags is a list + like ["--in0-csv-file", "...", "--in0-shape", "Dx...xD", ...] + and files is the list of created paths. + """ + out_dir = pathlib.Path(out_dir) + out_dir.mkdir(parents=True, exist_ok=True) + + ins, _ = onnx_io(onnx_path) + flags = [] + files = [] + for idx, _name, _et, shape in ins: + arr = arrays_in_order[idx] + csv_path = out_dir / f"in{idx}.csv" + # Write row-major flattened values, comma-separated, with newlines allowed + with open(csv_path, "w", newline="") as f: + writer = csv.writer(f) + # For 2D, write each row; otherwise write flattened single row for clarity + if arr.ndim == 2: + for r in range(arr.shape[0]): + writer.writerow(arr[r].reshape(-1)) + else: + writer.writerow(arr.flatten()) + + shape_str = "x".join(str(d) for d in arr.shape) + flags += [f"--in{idx}-csv-file", str(csv_path), f"--in{idx}-shape", shape_str] + files.append(str(csv_path)) + return flags, files + + +def write_inputs_to_memory_bin(memory_bin_path, config_json_path, arrays_in_order): + """Overwrite input regions in memory.bin at addresses from config.json.""" + with open(config_json_path) as f: + config = json.load(f) + + input_addresses = config["inputs_addresses"] + assert len(input_addresses) == len(arrays_in_order), \ + f"Address/input count mismatch: {len(input_addresses)} vs {len(arrays_in_order)}" + + with open(memory_bin_path, "r+b") as f: + for addr, arr in zip(input_addresses, arrays_in_order): + native = arr.astype(arr.dtype.newbyteorder("="), copy=False) + f.seek(addr) + f.write(native.tobytes(order="C")) diff --git a/validation/operations/gemm/simple/GemmTest5.onnx b/validation/operations/gemm/simple/GemmTest5.onnx new file mode 100644 index 0000000000000000000000000000000000000000..4d7ab1e060ef335ee2ab7e6cd607bbf09fa7b2c9 GIT binary patch literal 69804 zcmeIwF=_%)6a`S9nSt+z;P8{e+R`Q11Z={j4h&ifmWtrmTeuLDmAC`9lvrE1fdlWp z;+B__(){gwxs+2-W>Yb?NX(4{9hA`=)SI(YA`r`hz%HE zfB^;=V1NMz7+`<_1{h#~0R|XgfB^;=V1NMz7+`<_1{h#~0R|XgfB^;=V1NMz7+`<_ z1{h#~0R|XgfB^;=V1NMz7+`<_1{h#~0R|XgfB^;=V1NMz7+`<_1{h#~0R|XgfB^;= zV1NMz7+`<_1{h#~0R|XgfB^;=V1NMz7+`<_1{h#~0R|XgfB^;=V1NMz7+`<_1{h#~ z0R|XgfB^;=V1NMz7+`<_1{h#~0R|XgfB^;=V1NMz7+`<_1{h#~0R|XgfB^;=V1NMz z7+`<_1{h#~0R|XgfB^;=V1NMz7+`<_1{h#~0R|XgfB^;=V1NMz7+`<_1{h#~0R|Xg zfB^;=V1NMz7+`<_1{h#~0R|XgfB^;=V1NMz7+`<_1{h#~0R|XgfB^;=V1NMz7+`<_ z1{h#~0R|XgfB^;=V1NMz7+`<_2A&yMO>*7t_Q$@A^J|JS%EJ`-C7omao@4pYE{09{ E177CeT>t<8 literal 0 HcmV?d00001 diff --git a/validation/raptor.py b/validation/raptor.py new file mode 100644 index 0000000..e51a156 --- /dev/null +++ b/validation/raptor.py @@ -0,0 +1,29 @@ +import subprocess +from pathlib import Path +from colorama import Fore, Style + + +def compile_with_raptor(network_path, raptor_onnx_path: Path, crossbar_size=64, crossbar_count=16): + # Define the arguments, with the possibility to set crossbar size and count + args = [ + network_path, + "--maccel=PIM", + "--EmitPIMJSON", + "--EmitPimCodegen", + # "--use-experimental-conv-impl=true", + f"--crossbar-size={crossbar_size}", + f"--crossbar-count={crossbar_count}", + ] + + # Run the executable with the arguments + try: + result = subprocess.run( + [str(raptor_onnx_path)] + [str(arg) for arg in args], + check=True, + capture_output=True, + text=True, + ) + print(result.stdout + Fore.GREEN + "Raptor execution successful" + Style.RESET_ALL) + except subprocess.CalledProcessError as e: + print(Fore.RED + "Error executing ONNX-MLIR:") + print(e.stderr + Style.RESET_ALL) diff --git a/validation/validate.py b/validation/validate.py new file mode 100644 index 0000000..5c57893 --- /dev/null +++ b/validation/validate.py @@ -0,0 +1,135 @@ +import argparse +import json +import numpy as np +import subprocess +from pathlib import Path +from colorama import Style, Fore +from onnx_utils import gen_random_inputs, save_inputs_to_files, onnx_io, write_inputs_to_memory_bin, _ONNX_TO_NP +from raptor import compile_with_raptor +from gen_network_runner import gen_network_runner + + +def compile_onnx_network(network_onnx_path, raptor_path, raptor_dir, runner_dir): + subprocess.run([raptor_path, network_onnx_path, "--EmitONNXIR"], check=True) + subprocess.run([raptor_path, network_onnx_path], check=True) + parent = network_onnx_path.parent + stem = network_onnx_path.stem + so_path = parent / f"{stem}.so" + mlir_path = parent / f"{stem}.onnx.mlir" + tmp_path = parent / f"{stem}.tmp" + moved_so = runner_dir / so_path.name + moved_mlir = raptor_dir / mlir_path.name + so_path.rename(moved_so) + mlir_path.rename(moved_mlir) + tmp_path.unlink(missing_ok=True) + return moved_so, moved_mlir + + +def build_onnx_runner(source_dir, build_dir): + subprocess.run(["cmake", source_dir], cwd=build_dir, check=True) + subprocess.run(["cmake", "--build", ".", "-j"], cwd=build_dir, check=True) + return build_dir / "runner" + + +def build_dump_ranges(config_path, outputs_descriptor): + with open(config_path) as f: + output_addresses = json.load(f)["outputs_addresses"] + ranges = [] + for addr, (_, _, dtype_code, shape) in zip(output_addresses, outputs_descriptor): + byte_size = int(np.prod(shape)) * np.dtype(_ONNX_TO_NP[dtype_code]).itemsize + ranges.append(f"{addr},{byte_size}") + return ",".join(ranges) + + +def run_pim_simulator(simulator_dir, pim_dir, output_bin_path, dump_ranges): + subprocess.run( + ["cargo", "run", "--package", "pim-simulator", "--bin", "pim-simulator", "--", + "-f", str(pim_dir), "-o", str(output_bin_path), "-d", dump_ranges], + cwd=simulator_dir, check=True + ) + + +def parse_pim_simulator_outputs(output_bin_path, outputs_descriptor): + raw = output_bin_path.read_bytes() + arrays = [] + offset = 0 + for _, _, dtype_code, shape in outputs_descriptor: + dtype = np.dtype(_ONNX_TO_NP[dtype_code]) + count = int(np.prod(shape)) + array = np.frombuffer(raw, dtype=dtype, count=count, offset=offset).reshape(shape) + offset += count * dtype.itemsize + arrays.append(array) + return arrays + + +def validate_outputs(sim_arrays, runner_out_dir, outputs_descriptor, threshold=1e-5): + all_passed = True + for sim_array, (oi, name, _, shape) in zip(sim_arrays, outputs_descriptor): + csv_name = f"output{oi}_{name}.csv" + runner_array = np.loadtxt(runner_out_dir / csv_name, delimiter=',', dtype=np.float32).reshape(shape) + max_diff = float(np.max(np.abs(sim_array.astype(np.float64) - runner_array.astype(np.float64)))) + passed = max_diff <= threshold + status = Fore.GREEN + "[PASS]" if passed else Fore.RED + "[FAIL]" + print(f" {name}: max diff = {max_diff:.6e} {status}" + Style.RESET_ALL) + if not passed: + all_passed = False + return all_passed + + +if __name__ == '__main__': + ap = argparse.ArgumentParser() + ap.add_argument("--network-onnx", required=True) + ap.add_argument("--raptor-path", required=True) + ap.add_argument("--onnx-include-dir", required=True) + a = ap.parse_args() + + network_onnx_path = Path(a.network_onnx).absolute() + raptor_path = a.raptor_path + + workspace_dir = network_onnx_path.parent + raptor_dir = workspace_dir / "raptor" + runner_dir = workspace_dir / "runner" + runner_build_dir = runner_dir / "build" + Path.mkdir(raptor_dir, exist_ok=True) + Path.mkdir(runner_build_dir, parents=True, exist_ok=True) + + print(Style.BRIGHT + "\nCompiling the onnx network:" + Style.RESET_ALL) + network_so_path, network_mlir_path = compile_onnx_network(network_onnx_path, raptor_path, raptor_dir, runner_dir) + + print(Style.BRIGHT + "\nGenerating and building the runner:" + Style.RESET_ALL) + gen_network_runner(network_onnx_path, network_so_path, a.onnx_include_dir, out=runner_dir / "runner.c") + runner_path = build_onnx_runner(runner_dir, runner_build_dir) + + print(Style.BRIGHT + "\nGenerating random inputs:" + Style.RESET_ALL) + inputs_descriptor, outputs_descriptor = onnx_io(network_onnx_path) + inputs_list, inputs_dict = gen_random_inputs(inputs_descriptor) + flags, _files = save_inputs_to_files(network_onnx_path, inputs_list, out_dir=workspace_dir / "inputs") + + print(Style.BRIGHT + "\nRunning inference with the runner:" + Style.RESET_ALL) + out_dir = workspace_dir / "outputs" + Path.mkdir(out_dir, exist_ok=True) + run_cmd = [runner_path, *flags] + run_cmd += ["--save-csv-dir", f"{out_dir}"] + subprocess.run(run_cmd, cwd=runner_build_dir, check=True) + + # Configuration parameters. + matrix_height = matrix_width = 512 + crossbar_size = 64 + crossbar_count = 8 + + print(Style.BRIGHT + "Compiling for PIM with Raptor:" + Style.RESET_ALL) + compile_with_raptor(network_mlir_path, raptor_path, crossbar_size, crossbar_count) + + print(Style.BRIGHT + "\nRunning PIM simulation:" + Style.RESET_ALL) + pim_dir = raptor_dir / "pim" + write_inputs_to_memory_bin(pim_dir / "memory.bin", pim_dir / "config.json", inputs_list) + simulator_dir = Path(__file__).parent.resolve() / ".." / "backend-simulators" / "pim" / "pim-simulator" + simulation_dir = workspace_dir / "simulation" + Path.mkdir(simulation_dir, exist_ok=True) + dump_ranges = build_dump_ranges(pim_dir / "config.json", outputs_descriptor) + output_bin_path = simulation_dir / "out.bin" + run_pim_simulator(simulator_dir, pim_dir, output_bin_path, dump_ranges) + + print(Style.BRIGHT + "\nValidating the results:" + Style.RESET_ALL) + sim_arrays = parse_pim_simulator_outputs(output_bin_path, outputs_descriptor) + validate_outputs(sim_arrays, out_dir, outputs_descriptor)