add automatic validation process wit pim simulator
This commit is contained in:
NiccoloN
Submodule onnx-mlir updated: 840d057520...84cedd1d69
5
validation/.gitignore
vendored
Normal file
5
validation/.gitignore
vendored
Normal file
@@ -0,0 +1,5 @@
|
||||
operations/**/inputs
|
||||
operations/**/outputs
|
||||
operations/**/raptor
|
||||
operations/**/runner
|
||||
operations/**/simulation
|
||||
274
validation/gen_network_runner.py
Normal file
274
validation/gen_network_runner.py
Normal file
@@ -0,0 +1,274 @@
|
||||
#!/usr/bin/env python3
|
||||
|
||||
import argparse, os, pathlib, textwrap
|
||||
from onnx_utils import onnx_io
|
||||
from onnx import TensorProto
|
||||
|
||||
# ONNX dtype -> (ctype, printf, ONNX_TYPE_*)
|
||||
DTYPES = {
|
||||
TensorProto.FLOAT: ("float", "%g", "ONNX_TYPE_FLOAT"),
|
||||
TensorProto.DOUBLE: ("double", "%g", "ONNX_TYPE_DOUBLE"),
|
||||
TensorProto.INT64: ("int64_t", "%lld","ONNX_TYPE_INT64"),
|
||||
TensorProto.INT32: ("int32_t", "%d", "ONNX_TYPE_INT32"),
|
||||
TensorProto.UINT8: ("uint8_t", "%u", "ONNX_TYPE_UINT8"),
|
||||
TensorProto.INT8: ("int8_t", "%d", "ONNX_TYPE_INT8"),
|
||||
TensorProto.BOOL: ("uint8_t", "%u", "ONNX_TYPE_BOOL"), # stored as byte
|
||||
TensorProto.FLOAT16: ("uint16_t", "%u", "ONNX_TYPE_FLOAT16"), # raw 16-bit
|
||||
TensorProto.BFLOAT16:("uint16_t", "%u", "ONNX_TYPE_BFLOAT16"),
|
||||
}
|
||||
|
||||
def esc(s): return s.replace("\\","\\\\").replace('"','\\"')
|
||||
|
||||
def gen_c(inputs, outputs, entry, so_name):
|
||||
in_blocks=[]
|
||||
for i,name,et,shape in inputs:
|
||||
if et not in DTYPES:
|
||||
raise ValueError(f"Unsupported dtype for input '{name}': {et}")
|
||||
cty, pfmt, onnx_ty = DTYPES[et]
|
||||
shp_list = ", ".join(str(d) for d in shape) if shape else ""
|
||||
rank = len(shape)
|
||||
in_blocks.append(textwrap.dedent(f"""
|
||||
// ---- Input {i}: "{esc(name)}" ({cty}) ----
|
||||
const char *in{i}_csv=NULL, *in{i}_csv_file=NULL, *in{i}_shape_str=NULL;
|
||||
char *in{i}_csv_buf=NULL; // holds file contents if --in{i}-csv-file used
|
||||
int has_in{i}=0; double in{i}_fill=0.0; int in{i}_fill_set=0;
|
||||
|
||||
for (int ai=1; ai<argc; ++ai) {{
|
||||
if (strncmp(argv[ai],"--in{i}-csv",11)==0 && ai+1<argc) {{ in{i}_csv=argv[ai+1]; has_in{i}=1; }}
|
||||
else if (strncmp(argv[ai],"--in{i}-csv-file",16)==0 && ai+1<argc) {{ in{i}_csv_file=argv[ai+1]; has_in{i}=1; }}
|
||||
else if (strncmp(argv[ai],"--in{i}-fill",12)==0 && ai+1<argc) {{ in{i}_fill=atof(argv[ai+1]); in{i}_fill_set=1; has_in{i}=1; }}
|
||||
else if (strncmp(argv[ai],"--in{i}-shape",13)==0 && ai+1<argc) {{ in{i}_shape_str=argv[ai+1]; }}
|
||||
}}
|
||||
if (!has_in{i}) {{
|
||||
fprintf(stderr,"ERROR: provide one of --in{i}-csv/--in{i}-csv-file/--in{i}-fill for input {i}.\\n");
|
||||
return 2;
|
||||
}}
|
||||
|
||||
// If a CSV file was provided, read it fully and use its content as the CSV string.
|
||||
if (in{i}_csv_file && !in{i}_csv) {{
|
||||
FILE *f=fopen(in{i}_csv_file,"rb");
|
||||
if(!f){{perror("fopen --in{i}-csv-file"); return 2;}}
|
||||
fseek(f, 0, SEEK_END);
|
||||
long sz = ftell(f); if (sz < 0) {{ perror("ftell"); fclose(f); return 2; }}
|
||||
fseek(f, 0, SEEK_SET);
|
||||
in{i}_csv_buf = (char*)malloc((size_t)sz + 1);
|
||||
if(!in{i}_csv_buf){{fprintf(stderr,"OOM reading --in{i}-csv-file.\\n"); fclose(f); return 2;}}
|
||||
size_t got = fread(in{i}_csv_buf, 1, (size_t)sz, f);
|
||||
fclose(f);
|
||||
if (got != (size_t)sz) {{ fprintf(stderr,"ERROR: short read for --in{i}-csv-file.\\n"); free(in{i}_csv_buf); return 2; }}
|
||||
in{i}_csv_buf[sz] = '\\0';
|
||||
in{i}_csv = in{i}_csv_buf;
|
||||
}}
|
||||
|
||||
int64_t *in{i}_shape=NULL; int in{i}_rank=0;
|
||||
if (in{i}_shape_str) {{
|
||||
char *tmp=strdup(in{i}_shape_str);
|
||||
for(char*p=tmp; *p; ++p) if(*p=='x'||*p=='X') in{i}_rank++;
|
||||
in{i}_rank++;
|
||||
in{i}_shape=(int64_t*)malloc(sizeof(int64_t)*in{i}_rank);
|
||||
int di=0; char *tok=strtok(tmp,"xX");
|
||||
while(tok && di<in{i}_rank) {{ in{i}_shape[di++]=atoll(tok); tok=strtok(NULL,"xX"); }}
|
||||
free(tmp);
|
||||
}} else {{
|
||||
in{i}_rank={rank};
|
||||
in{i}_shape=(int64_t*)malloc(sizeof(int64_t)*in{i}_rank);
|
||||
int64_t def_shape[]={{{shp_list}}};
|
||||
for(int k=0;k<in{i}_rank;k++) in{i}_shape[k]=def_shape[k];
|
||||
}}
|
||||
long long in{i}_nelem=1; for(int k=0;k<in{i}_rank;k++) in{i}_nelem*=in{i}_shape[k];
|
||||
|
||||
size_t in{i}_bytes = sizeof({cty}) * (size_t)in{i}_nelem;
|
||||
void *in{i}_buf = malloc(in{i}_bytes);
|
||||
if(!in{i}_buf){{fprintf(stderr,"OOM for input {i}.\\n"); if(in{i}_csv_buf) free(in{i}_csv_buf); return 2;}}
|
||||
|
||||
if (in{i}_csv) {{
|
||||
char *buf=strdup(in{i}_csv); long long idx=0; char *tok=strtok(buf,",\\n\\r\\t ");
|
||||
while(tok) {{
|
||||
if(idx>=in{i}_nelem) break;
|
||||
double v=atof(tok);
|
||||
(({cty}*)in{i}_buf)[idx++] = ({cty})v;
|
||||
tok=strtok(NULL,",\\n\\r\\t ");
|
||||
}}
|
||||
free(buf);
|
||||
if(idx!=in{i}_nelem){{fprintf(stderr,"ERROR: CSV provided %lld values, expected %lld.\\n",(long long)idx,in{i}_nelem); if(in{i}_csv_buf) free(in{i}_csv_buf); return 2;}}
|
||||
}} else if (in{i}_fill_set) {{
|
||||
{cty} vv=({cty})in{i}_fill; for(long long t=0;t<in{i}_nelem;t++) (({cty}*)in{i}_buf)[t]=vv;
|
||||
}} else {{
|
||||
fprintf(stderr,"ERROR: no data source for input {i}.\\n"); if(in{i}_csv_buf) free(in{i}_csv_buf); return 2;
|
||||
}}
|
||||
|
||||
OMTensor *in{i}_tensor = omTensorCreateWithOwnership(in{i}_buf, in{i}_shape, in{i}_rank, {onnx_ty}, /*owning=*/1);
|
||||
if(in{i}_csv_buf) free(in{i}_csv_buf);
|
||||
if(!in{i}_tensor){{fprintf(stderr,"ERROR: omTensorCreateWithOwnership failed for input {i}.\\n");return 2;}}
|
||||
"""))
|
||||
|
||||
# Output printing + optional per-output CSV dump
|
||||
out_blocks=[]
|
||||
csv_write_blocks=[]
|
||||
for oi,name,et,shape in outputs:
|
||||
if et not in DTYPES:
|
||||
raise ValueError(f"Unsupported dtype for output '{name}': {et}")
|
||||
cty, pfmt, _ = DTYPES[et]
|
||||
safe = esc(name)
|
||||
out_blocks.append(textwrap.dedent(f"""
|
||||
// ---- Output {oi}: "{safe}" ----
|
||||
{{
|
||||
OMTensor *t = omTensorListGetOmtByIndex(out_list, {oi});
|
||||
int64_t rank = omTensorGetRank(t);
|
||||
int64_t const *shape = omTensorGetShape(t);
|
||||
long long numel = 1; for (int64_t k=0;k<rank;k++) numel *= shape[k];
|
||||
{cty} *p = ({cty}*)omTensorGetDataPtr(t);
|
||||
|
||||
printf("Output {oi} ('{safe}'): shape=[");
|
||||
for (int64_t k=0;k<rank;k++) printf("%ld%s",(long)shape[k], (k+1<rank)?",":"");
|
||||
printf("]\\n");
|
||||
|
||||
if (rank == 2) {{
|
||||
int64_t R = shape[0], C = shape[1];
|
||||
for (int64_t r=0; r<R; ++r) {{
|
||||
for (int64_t c=0; c<C; ++c) {{
|
||||
long long idx = r*C + c;
|
||||
printf("{pfmt}%s", p[idx], (c+1<C)?", ":"");
|
||||
}}
|
||||
printf("\\n");
|
||||
}}
|
||||
}} else {{
|
||||
// Flattened vector with indices
|
||||
for (long long i=0;i<numel;i++) {{
|
||||
printf("[%lld]={pfmt}%s", i, p[i], (i+1<numel)?", ":"\\n");
|
||||
}}
|
||||
}}
|
||||
}}
|
||||
"""))
|
||||
|
||||
# Per-output CSV writer into --save-csv-dir
|
||||
csv_write_blocks.append(textwrap.dedent(f"""
|
||||
if (save_csv_dir) {{
|
||||
// Build "DIR/output{oi}_<sanitized name>.csv"
|
||||
char fname[512];
|
||||
// simple sanitizer: copy name => replace non [A-Za-z0-9_.-] with '_'
|
||||
char clean[256]; int ci=0; const char *src="{safe}";
|
||||
for (; src[ci] && ci < 255; ++ci) {{
|
||||
char ch = src[ci];
|
||||
int ok = (ch>='A'&&ch<='Z')||(ch>='a'&&ch<='z')||(ch>='0'&&ch<='9')||ch=='_'||ch=='-'||ch=='.';
|
||||
clean[ci] = ok ? ch : '_';
|
||||
}}
|
||||
clean[ci] = '\\0';
|
||||
snprintf(fname, sizeof(fname), "%s/output{oi}_%s.csv", save_csv_dir, clean);
|
||||
FILE *csv = fopen(fname, "w");
|
||||
if (!csv) {{ perror("fopen --save-csv-dir"); }}
|
||||
else {{
|
||||
OMTensor *t = omTensorListGetOmtByIndex(out_list, {oi});
|
||||
int64_t rank = omTensorGetRank(t);
|
||||
int64_t const *shape = omTensorGetShape(t);
|
||||
long long numel = 1; for (int64_t k=0;k<rank;k++) numel *= shape[k];
|
||||
{cty} *p = ({cty}*)omTensorGetDataPtr(t);
|
||||
|
||||
if (rank == 2) {{
|
||||
int64_t R = shape[0], C = shape[1];
|
||||
for (int64_t r=0; r<R; ++r) {{
|
||||
for (int64_t c=0; c<C; ++c) {{
|
||||
long long idx = r*C + c;
|
||||
fprintf(csv, "{pfmt}%s", p[idx], (c+1<C)?",":"");
|
||||
}}
|
||||
fprintf(csv, "\\n");
|
||||
}}
|
||||
}} else {{
|
||||
for (long long i=0;i<numel;i++) {{
|
||||
fprintf(csv, "{pfmt}%s", p[i], (i+1<numel)?",":"");
|
||||
}}
|
||||
fprintf(csv, "\\n");
|
||||
}}
|
||||
fclose(csv);
|
||||
}}
|
||||
}}
|
||||
"""))
|
||||
|
||||
n_in=len(inputs)
|
||||
build_inputs="\n".join([f" arr[{i}] = in{i}_tensor;" for i,_,_,_ in inputs])
|
||||
|
||||
return f"""\
|
||||
// Auto-generated onnx network runner
|
||||
#include "OnnxMlirRuntime.h"
|
||||
#include <stdio.h>
|
||||
#include <stdlib.h>
|
||||
#include <string.h>
|
||||
#include <inttypes.h>
|
||||
|
||||
OMTensorList *{entry}(OMTensorList *inputs);
|
||||
|
||||
int main(int argc, char **argv) {{
|
||||
// optional: --save-csv-dir <DIR> (directory must exist)
|
||||
const char *save_csv_dir = NULL;
|
||||
for (int ai=1; ai<argc; ++ai) {{
|
||||
if (strncmp(argv[ai], "--save-csv-dir", 14)==0 && ai+1 < argc) {{
|
||||
save_csv_dir = argv[ai+1];
|
||||
}}
|
||||
}}
|
||||
|
||||
if (argc == 1) {{
|
||||
fprintf(stderr,
|
||||
"Usage: %s "
|
||||
"[--inK-csv \\\"v1,v2,...\\\" | --inK-csv-file path | --inK-fill c] "
|
||||
"[--inK-shape 1x...xD] "
|
||||
"[--save-csv-dir /path/to/dir]\\n"
|
||||
"Repeat for K=0..%d.\\n",
|
||||
argv[0], {max(0, n_in-1)});
|
||||
return 1;
|
||||
}}
|
||||
|
||||
{"".join(in_blocks)}
|
||||
|
||||
OMTensor *arr[{n_in}];
|
||||
{build_inputs}
|
||||
OMTensorList *in_list = omTensorListCreate(arr, {n_in});
|
||||
if(!in_list){{fprintf(stderr,"ERROR: omTensorListCreate failed.\\n");return 2;}}
|
||||
|
||||
OMTensorList *out_list = {entry}(in_list);
|
||||
if(!out_list){{fprintf(stderr,"ERROR: model returned NULL.\\n");omTensorListDestroy(in_list);return 3;}}
|
||||
|
||||
// ---- Print full outputs ----
|
||||
{"".join(out_blocks)}
|
||||
|
||||
// ---- Optional per-output CSV dump ----
|
||||
{"".join(csv_write_blocks)}
|
||||
|
||||
// ---- Cleanup ----
|
||||
omTensorListDestroy(in_list);
|
||||
omTensorListDestroy(out_list);
|
||||
return 0;
|
||||
}}
|
||||
"""
|
||||
|
||||
def gen_network_runner(network_onnx, network_so, onnx_include_dir, entry="run_main_graph", out=None):
|
||||
ins, outs = onnx_io(network_onnx)
|
||||
out_c = out or "runner.c"
|
||||
so_abs = os.path.abspath(network_so)
|
||||
|
||||
csrc = gen_c(ins, outs, entry, pathlib.Path(so_abs).name)
|
||||
pathlib.Path(out_c).write_text(csrc)
|
||||
|
||||
cmake=f"""\
|
||||
cmake_minimum_required(VERSION 3.15)
|
||||
project(onnx_mlir_runner C)
|
||||
add_executable({pathlib.Path(out_c).stem} {pathlib.Path(out_c).name})
|
||||
target_include_directories({pathlib.Path(out_c).stem} PUBLIC {esc(onnx_include_dir)})
|
||||
|
||||
add_library(model_so SHARED IMPORTED)
|
||||
set_target_properties(model_so PROPERTIES IMPORTED_LOCATION {esc(so_abs)})
|
||||
target_link_libraries({pathlib.Path(out_c).stem} PUBLIC model_so)
|
||||
"""
|
||||
pathlib.Path(out_c).with_name("CMakeLists.txt").write_text(cmake)
|
||||
print(f"[OK] Wrote {out_c}")
|
||||
print("[OK] Wrote CMakeLists.txt")
|
||||
|
||||
if __name__=="__main__":
|
||||
ap=argparse.ArgumentParser()
|
||||
ap.add_argument("--network-onnx", required=True)
|
||||
ap.add_argument("--network-so", required=True)
|
||||
ap.add_argument("--onnx-include-dir", required=True)
|
||||
ap.add_argument("--entry", default="run_main_graph")
|
||||
ap.add_argument("--out", default=None)
|
||||
a=ap.parse_args()
|
||||
|
||||
gen_network_runner(a.network_onnx, a.network_so, a.onnx_include_dir, a.entry, a.out)
|
||||
200
validation/onnx_utils.py
Normal file
200
validation/onnx_utils.py
Normal file
@@ -0,0 +1,200 @@
|
||||
import csv
|
||||
import onnx
|
||||
import json
|
||||
import pathlib
|
||||
import numpy as np
|
||||
from onnx import TensorProto
|
||||
|
||||
_ONNX_TO_NP = {
|
||||
TensorProto.FLOAT: np.float32,
|
||||
TensorProto.DOUBLE: np.float64,
|
||||
TensorProto.INT64: np.int64,
|
||||
TensorProto.INT32: np.int32,
|
||||
TensorProto.UINT8: np.uint8,
|
||||
TensorProto.INT8: np.int8,
|
||||
TensorProto.BOOL: np.uint8, # store as 0/1 bytes
|
||||
TensorProto.FLOAT16: np.float16, # generate in f32 then cast
|
||||
TensorProto.BFLOAT16: getattr(np, "bfloat16", np.float32), # cast if available
|
||||
}
|
||||
|
||||
|
||||
def onnx_io(path):
|
||||
m = onnx.load(path)
|
||||
g = m.graph
|
||||
|
||||
def shp(tt):
|
||||
s = []
|
||||
if tt.HasField("shape"):
|
||||
for d in tt.shape.dim:
|
||||
s.append(int(d.dim_value) if d.HasField("dim_value") else 1)
|
||||
return s
|
||||
|
||||
ins, outs = [], []
|
||||
for i, v in enumerate(g.input):
|
||||
t = v.type.tensor_type
|
||||
ins.append((i, v.name, t.elem_type, shp(t)))
|
||||
for i, v in enumerate(g.output):
|
||||
t = v.type.tensor_type
|
||||
outs.append((i, v.name, t.elem_type, shp(t)))
|
||||
return ins, outs
|
||||
|
||||
|
||||
def onnx_io_bitsize(io):
|
||||
idx, name, elem_type, shape = io
|
||||
num_elements = shape[0]
|
||||
for dim in shape[1:]:
|
||||
num_elements *= dim
|
||||
return num_elements * _ONNX_TO_NP[elem_type]().itemsize * 8
|
||||
|
||||
|
||||
def _dtype_bounds(np_dtype):
|
||||
"""Return (min, max) inclusive bounds for integer dtypes; None for floats."""
|
||||
if np_dtype in (np.int8, np.int16, np.int32, np.int64):
|
||||
info = np.iinfo(np_dtype)
|
||||
return int(info.min), int(info.max)
|
||||
if np_dtype in (np.uint8, np.uint16, np.uint32, np.uint64):
|
||||
info = np.iinfo(np_dtype)
|
||||
return int(info.min), int(info.max)
|
||||
return None
|
||||
|
||||
|
||||
def gen_random_inputs(
|
||||
onnx_inputs,
|
||||
*,
|
||||
shape_overrides: dict | None = None,
|
||||
float_range: tuple[float, float] = (-1.0, 1.0),
|
||||
int_range: tuple[int, int] = (-3, 3),
|
||||
dyn_dim_default: int = 1,
|
||||
seed: int | None = None,
|
||||
):
|
||||
"""
|
||||
Generate random NumPy arrays for each ONNX input.
|
||||
|
||||
Params
|
||||
------
|
||||
shape_overrides:
|
||||
Dict mapping input index OR input name -> tuple/list of dims.
|
||||
Overrides the shape inferred from the model (useful for dynamic dims).
|
||||
float_range:
|
||||
Range for floats (uniform).
|
||||
int_range:
|
||||
Range for integers (uniform integers, inclusive of low/high with np.integers semantics).
|
||||
dyn_dim_default:
|
||||
If a dim is dynamic/unknown, use this value (unless shape_overrides provides one).
|
||||
seed:
|
||||
RNG seed for reproducibility.
|
||||
|
||||
Returns
|
||||
-------
|
||||
inputs_list: list[np.ndarray]
|
||||
Arrays in graph input order (index-sorted).
|
||||
inputs_dict: dict[str, np.ndarray]
|
||||
Mapping input_name -> array in the ONNX-declared dtype.
|
||||
"""
|
||||
rng = np.random.default_rng(seed)
|
||||
ins = onnx_inputs
|
||||
|
||||
# Normalize overrides to support both index and name keys.
|
||||
shape_overrides = shape_overrides or {}
|
||||
name_overrides = {k: tuple(v) for k, v in shape_overrides.items() if isinstance(k, str)}
|
||||
idx_overrides = {int(k): tuple(v) for k, v in shape_overrides.items() if isinstance(k, int)}
|
||||
|
||||
arrays_by_name = {}
|
||||
arrays_in_order = []
|
||||
|
||||
for idx, name, elem_type, shape in ins:
|
||||
# Resolve dtype
|
||||
if elem_type not in _ONNX_TO_NP:
|
||||
raise ValueError(f"Unsupported ONNX dtype for input '{name}': {elem_type}")
|
||||
np_dtype = _ONNX_TO_NP[elem_type]
|
||||
|
||||
# Resolve shape: model -> replace unknowns with dyn_dim_default -> apply overrides
|
||||
resolved_shape = list(shape or [])
|
||||
if not resolved_shape:
|
||||
resolved_shape = [dyn_dim_default] # scalar-like: treat as 1-dim with size dyn_dim_default
|
||||
|
||||
# If your onnx_io already sets unknown dims to 1, we still allow overriding:
|
||||
if idx in idx_overrides:
|
||||
resolved_shape = list(idx_overrides[idx])
|
||||
elif name in name_overrides:
|
||||
resolved_shape = list(name_overrides[name])
|
||||
|
||||
# Make sure no zeros
|
||||
resolved_shape = [int(d if d and d > 0 else dyn_dim_default) for d in resolved_shape]
|
||||
size = int(np.prod(resolved_shape))
|
||||
|
||||
# Generate data
|
||||
if np.issubdtype(np_dtype, np.floating):
|
||||
lo, hi = float_range
|
||||
# generate in float32/64 and cast as needed
|
||||
base_dtype = np.float32 if np_dtype in (np.float16, getattr(np, "bfloat16", np.float32)) else np_dtype
|
||||
arr = rng.uniform(lo, hi, size=size).astype(base_dtype).reshape(resolved_shape)
|
||||
# cast to f16/bf16 if required
|
||||
if np_dtype is np.float16:
|
||||
arr = arr.astype(np.float16)
|
||||
elif getattr(np, "bfloat16", None) is not None and np_dtype is np.bfloat16:
|
||||
arr = arr.astype(np.bfloat16)
|
||||
elif np_dtype == np.uint8 and elem_type == TensorProto.BOOL:
|
||||
# Bool as 0/1 bytes
|
||||
arr = (rng.random(size=size) < 0.5).astype(np.uint8).reshape(resolved_shape)
|
||||
elif np.issubdtype(np_dtype, np.integer):
|
||||
lo, hi = int_range
|
||||
bounds = _dtype_bounds(np_dtype)
|
||||
if bounds is not None:
|
||||
lo = max(lo, bounds[0])
|
||||
hi = min(hi, bounds[1])
|
||||
# np.random.integers is exclusive of high; add 1 for int range
|
||||
arr = rng.integers(lo, hi + 1, size=size, dtype=np_dtype).reshape(resolved_shape)
|
||||
else:
|
||||
raise ValueError(f"Unhandled dtype mapping for input '{name}' (elem_type={elem_type}).")
|
||||
|
||||
arrays_by_name[name] = arr
|
||||
arrays_in_order.append(arr)
|
||||
return arrays_in_order, arrays_by_name
|
||||
|
||||
|
||||
def save_inputs_to_files(onnx_path, arrays_in_order, out_dir):
|
||||
"""
|
||||
Save arrays to CSV files. Returns (flags, files) where flags is a list
|
||||
like ["--in0-csv-file", "...", "--in0-shape", "Dx...xD", ...]
|
||||
and files is the list of created paths.
|
||||
"""
|
||||
out_dir = pathlib.Path(out_dir)
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
ins, _ = onnx_io(onnx_path)
|
||||
flags = []
|
||||
files = []
|
||||
for idx, _name, _et, shape in ins:
|
||||
arr = arrays_in_order[idx]
|
||||
csv_path = out_dir / f"in{idx}.csv"
|
||||
# Write row-major flattened values, comma-separated, with newlines allowed
|
||||
with open(csv_path, "w", newline="") as f:
|
||||
writer = csv.writer(f)
|
||||
# For 2D, write each row; otherwise write flattened single row for clarity
|
||||
if arr.ndim == 2:
|
||||
for r in range(arr.shape[0]):
|
||||
writer.writerow(arr[r].reshape(-1))
|
||||
else:
|
||||
writer.writerow(arr.flatten())
|
||||
|
||||
shape_str = "x".join(str(d) for d in arr.shape)
|
||||
flags += [f"--in{idx}-csv-file", str(csv_path), f"--in{idx}-shape", shape_str]
|
||||
files.append(str(csv_path))
|
||||
return flags, files
|
||||
|
||||
|
||||
def write_inputs_to_memory_bin(memory_bin_path, config_json_path, arrays_in_order):
|
||||
"""Overwrite input regions in memory.bin at addresses from config.json."""
|
||||
with open(config_json_path) as f:
|
||||
config = json.load(f)
|
||||
|
||||
input_addresses = config["inputs_addresses"]
|
||||
assert len(input_addresses) == len(arrays_in_order), \
|
||||
f"Address/input count mismatch: {len(input_addresses)} vs {len(arrays_in_order)}"
|
||||
|
||||
with open(memory_bin_path, "r+b") as f:
|
||||
for addr, arr in zip(input_addresses, arrays_in_order):
|
||||
native = arr.astype(arr.dtype.newbyteorder("="), copy=False)
|
||||
f.seek(addr)
|
||||
f.write(native.tobytes(order="C"))
|
||||
BIN
validation/operations/gemm/simple/GemmTest5.onnx
Normal file
BIN
validation/operations/gemm/simple/GemmTest5.onnx
Normal file
Binary file not shown.
29
validation/raptor.py
Normal file
29
validation/raptor.py
Normal file
@@ -0,0 +1,29 @@
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from colorama import Fore, Style
|
||||
|
||||
|
||||
def compile_with_raptor(network_path, raptor_onnx_path: Path, crossbar_size=64, crossbar_count=16):
|
||||
# Define the arguments, with the possibility to set crossbar size and count
|
||||
args = [
|
||||
network_path,
|
||||
"--maccel=PIM",
|
||||
"--EmitPIMJSON",
|
||||
"--EmitPimCodegen",
|
||||
# "--use-experimental-conv-impl=true",
|
||||
f"--crossbar-size={crossbar_size}",
|
||||
f"--crossbar-count={crossbar_count}",
|
||||
]
|
||||
|
||||
# Run the executable with the arguments
|
||||
try:
|
||||
result = subprocess.run(
|
||||
[str(raptor_onnx_path)] + [str(arg) for arg in args],
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
print(result.stdout + Fore.GREEN + "Raptor execution successful" + Style.RESET_ALL)
|
||||
except subprocess.CalledProcessError as e:
|
||||
print(Fore.RED + "Error executing ONNX-MLIR:")
|
||||
print(e.stderr + Style.RESET_ALL)
|
||||
135
validation/validate.py
Normal file
135
validation/validate.py
Normal file
@@ -0,0 +1,135 @@
|
||||
import argparse
|
||||
import json
|
||||
import numpy as np
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from colorama import Style, Fore
|
||||
from onnx_utils import gen_random_inputs, save_inputs_to_files, onnx_io, write_inputs_to_memory_bin, _ONNX_TO_NP
|
||||
from raptor import compile_with_raptor
|
||||
from gen_network_runner import gen_network_runner
|
||||
|
||||
|
||||
def compile_onnx_network(network_onnx_path, raptor_path, raptor_dir, runner_dir):
|
||||
subprocess.run([raptor_path, network_onnx_path, "--EmitONNXIR"], check=True)
|
||||
subprocess.run([raptor_path, network_onnx_path], check=True)
|
||||
parent = network_onnx_path.parent
|
||||
stem = network_onnx_path.stem
|
||||
so_path = parent / f"{stem}.so"
|
||||
mlir_path = parent / f"{stem}.onnx.mlir"
|
||||
tmp_path = parent / f"{stem}.tmp"
|
||||
moved_so = runner_dir / so_path.name
|
||||
moved_mlir = raptor_dir / mlir_path.name
|
||||
so_path.rename(moved_so)
|
||||
mlir_path.rename(moved_mlir)
|
||||
tmp_path.unlink(missing_ok=True)
|
||||
return moved_so, moved_mlir
|
||||
|
||||
|
||||
def build_onnx_runner(source_dir, build_dir):
|
||||
subprocess.run(["cmake", source_dir], cwd=build_dir, check=True)
|
||||
subprocess.run(["cmake", "--build", ".", "-j"], cwd=build_dir, check=True)
|
||||
return build_dir / "runner"
|
||||
|
||||
|
||||
def build_dump_ranges(config_path, outputs_descriptor):
|
||||
with open(config_path) as f:
|
||||
output_addresses = json.load(f)["outputs_addresses"]
|
||||
ranges = []
|
||||
for addr, (_, _, dtype_code, shape) in zip(output_addresses, outputs_descriptor):
|
||||
byte_size = int(np.prod(shape)) * np.dtype(_ONNX_TO_NP[dtype_code]).itemsize
|
||||
ranges.append(f"{addr},{byte_size}")
|
||||
return ",".join(ranges)
|
||||
|
||||
|
||||
def run_pim_simulator(simulator_dir, pim_dir, output_bin_path, dump_ranges):
|
||||
subprocess.run(
|
||||
["cargo", "run", "--package", "pim-simulator", "--bin", "pim-simulator", "--",
|
||||
"-f", str(pim_dir), "-o", str(output_bin_path), "-d", dump_ranges],
|
||||
cwd=simulator_dir, check=True
|
||||
)
|
||||
|
||||
|
||||
def parse_pim_simulator_outputs(output_bin_path, outputs_descriptor):
|
||||
raw = output_bin_path.read_bytes()
|
||||
arrays = []
|
||||
offset = 0
|
||||
for _, _, dtype_code, shape in outputs_descriptor:
|
||||
dtype = np.dtype(_ONNX_TO_NP[dtype_code])
|
||||
count = int(np.prod(shape))
|
||||
array = np.frombuffer(raw, dtype=dtype, count=count, offset=offset).reshape(shape)
|
||||
offset += count * dtype.itemsize
|
||||
arrays.append(array)
|
||||
return arrays
|
||||
|
||||
|
||||
def validate_outputs(sim_arrays, runner_out_dir, outputs_descriptor, threshold=1e-5):
|
||||
all_passed = True
|
||||
for sim_array, (oi, name, _, shape) in zip(sim_arrays, outputs_descriptor):
|
||||
csv_name = f"output{oi}_{name}.csv"
|
||||
runner_array = np.loadtxt(runner_out_dir / csv_name, delimiter=',', dtype=np.float32).reshape(shape)
|
||||
max_diff = float(np.max(np.abs(sim_array.astype(np.float64) - runner_array.astype(np.float64))))
|
||||
passed = max_diff <= threshold
|
||||
status = Fore.GREEN + "[PASS]" if passed else Fore.RED + "[FAIL]"
|
||||
print(f" {name}: max diff = {max_diff:.6e} {status}" + Style.RESET_ALL)
|
||||
if not passed:
|
||||
all_passed = False
|
||||
return all_passed
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
ap = argparse.ArgumentParser()
|
||||
ap.add_argument("--network-onnx", required=True)
|
||||
ap.add_argument("--raptor-path", required=True)
|
||||
ap.add_argument("--onnx-include-dir", required=True)
|
||||
a = ap.parse_args()
|
||||
|
||||
network_onnx_path = Path(a.network_onnx).absolute()
|
||||
raptor_path = a.raptor_path
|
||||
|
||||
workspace_dir = network_onnx_path.parent
|
||||
raptor_dir = workspace_dir / "raptor"
|
||||
runner_dir = workspace_dir / "runner"
|
||||
runner_build_dir = runner_dir / "build"
|
||||
Path.mkdir(raptor_dir, exist_ok=True)
|
||||
Path.mkdir(runner_build_dir, parents=True, exist_ok=True)
|
||||
|
||||
print(Style.BRIGHT + "\nCompiling the onnx network:" + Style.RESET_ALL)
|
||||
network_so_path, network_mlir_path = compile_onnx_network(network_onnx_path, raptor_path, raptor_dir, runner_dir)
|
||||
|
||||
print(Style.BRIGHT + "\nGenerating and building the runner:" + Style.RESET_ALL)
|
||||
gen_network_runner(network_onnx_path, network_so_path, a.onnx_include_dir, out=runner_dir / "runner.c")
|
||||
runner_path = build_onnx_runner(runner_dir, runner_build_dir)
|
||||
|
||||
print(Style.BRIGHT + "\nGenerating random inputs:" + Style.RESET_ALL)
|
||||
inputs_descriptor, outputs_descriptor = onnx_io(network_onnx_path)
|
||||
inputs_list, inputs_dict = gen_random_inputs(inputs_descriptor)
|
||||
flags, _files = save_inputs_to_files(network_onnx_path, inputs_list, out_dir=workspace_dir / "inputs")
|
||||
|
||||
print(Style.BRIGHT + "\nRunning inference with the runner:" + Style.RESET_ALL)
|
||||
out_dir = workspace_dir / "outputs"
|
||||
Path.mkdir(out_dir, exist_ok=True)
|
||||
run_cmd = [runner_path, *flags]
|
||||
run_cmd += ["--save-csv-dir", f"{out_dir}"]
|
||||
subprocess.run(run_cmd, cwd=runner_build_dir, check=True)
|
||||
|
||||
# Configuration parameters.
|
||||
matrix_height = matrix_width = 512
|
||||
crossbar_size = 64
|
||||
crossbar_count = 8
|
||||
|
||||
print(Style.BRIGHT + "Compiling for PIM with Raptor:" + Style.RESET_ALL)
|
||||
compile_with_raptor(network_mlir_path, raptor_path, crossbar_size, crossbar_count)
|
||||
|
||||
print(Style.BRIGHT + "\nRunning PIM simulation:" + Style.RESET_ALL)
|
||||
pim_dir = raptor_dir / "pim"
|
||||
write_inputs_to_memory_bin(pim_dir / "memory.bin", pim_dir / "config.json", inputs_list)
|
||||
simulator_dir = Path(__file__).parent.resolve() / ".." / "backend-simulators" / "pim" / "pim-simulator"
|
||||
simulation_dir = workspace_dir / "simulation"
|
||||
Path.mkdir(simulation_dir, exist_ok=True)
|
||||
dump_ranges = build_dump_ranges(pim_dir / "config.json", outputs_descriptor)
|
||||
output_bin_path = simulation_dir / "out.bin"
|
||||
run_pim_simulator(simulator_dir, pim_dir, output_bin_path, dump_ranges)
|
||||
|
||||
print(Style.BRIGHT + "\nValidating the results:" + Style.RESET_ALL)
|
||||
sim_arrays = parse_pim_simulator_outputs(output_bin_path, outputs_descriptor)
|
||||
validate_outputs(sim_arrays, out_dir, outputs_descriptor)
|
||||
Reference in New Issue
Block a user