diff --git a/backend-simulators/pim/pim-simulator/src/bin/pim-simulator/main.rs b/backend-simulators/pim/pim-simulator/src/bin/pim-simulator/main.rs index 104f9ca..cdfef47 100644 --- a/backend-simulators/pim/pim-simulator/src/bin/pim-simulator/main.rs +++ b/backend-simulators/pim/pim-simulator/src/bin/pim-simulator/main.rs @@ -1,11 +1,10 @@ use anyhow::{Context, Result, bail}; use clap::Parser; use glob::glob; -use pimcore::cpu::crossbar; use pimcore::json_to_instruction::json_to_executor; use pimcore::tracing::TRACER; use serde_json::Value; -use std::{fs, usize}; +use std::fs; use std::io::Write; use std::path::PathBuf; diff --git a/validation/raptor.py b/validation/raptor.py index a3e4e53..1476b92 100644 --- a/validation/raptor.py +++ b/validation/raptor.py @@ -3,7 +3,7 @@ from pathlib import Path from colorama import Fore, Style -def compile_with_raptor(network_path, raptor_onnx_path: Path, crossbar_size=64, crossbar_count=16): +def compile_with_raptor(network_path, raptor_onnx_path: Path, crossbar_size, crossbar_count): # Define the arguments, with the possibility to set crossbar size and count args = [ network_path, diff --git a/validation/validate.py b/validation/validate.py index cea876a..5cb1a53 100644 --- a/validation/validate.py +++ b/validation/validate.py @@ -1,135 +1,76 @@ +#!/usr/bin/env python3 + import argparse -import json -import numpy as np -import subprocess +import sys from pathlib import Path from colorama import Style, Fore -from onnx_utils import gen_random_inputs, save_inputs_to_files, onnx_io, write_inputs_to_memory_bin, _ONNX_TO_NP -from raptor import compile_with_raptor -from gen_network_runner import gen_network_runner +from validate_one import validate_network -def compile_onnx_network(network_onnx_path, raptor_path, raptor_dir, runner_dir): - subprocess.run([raptor_path, network_onnx_path, "--EmitONNXIR"], check=True) - subprocess.run([raptor_path, network_onnx_path], check=True) - parent = network_onnx_path.parent - stem = network_onnx_path.stem - so_path = parent / f"{stem}.so" - mlir_path = parent / f"{stem}.onnx.mlir" - tmp_path = parent / f"{stem}.tmp" - moved_so = runner_dir / so_path.name - moved_mlir = raptor_dir / mlir_path.name - so_path.rename(moved_so) - mlir_path.rename(moved_mlir) - tmp_path.unlink(missing_ok=True) - return moved_so, moved_mlir +def discover_onnx_files(operations_dir): + return sorted(operations_dir.rglob("*.onnx")) -def build_onnx_runner(source_dir, build_dir): - subprocess.run(["cmake", source_dir], cwd=build_dir, check=True) - subprocess.run(["cmake", "--build", ".", "-j"], cwd=build_dir, check=True) - return build_dir / "runner" - - -def build_dump_ranges(config_path, outputs_descriptor): - with open(config_path) as f: - output_addresses = json.load(f)["outputs_addresses"] - ranges = [] - for addr, (_, _, dtype_code, shape) in zip(output_addresses, outputs_descriptor): - byte_size = int(np.prod(shape)) * np.dtype(_ONNX_TO_NP[dtype_code]).itemsize - ranges.append(f"{addr},{byte_size}") - return ",".join(ranges) - - -def run_pim_simulator(simulator_dir, pim_dir, output_bin_path, dump_ranges): - subprocess.run( - ["cargo", "run", "--release", "--package", "pim-simulator", "--bin", "pim-simulator", "--", - "-f", str(pim_dir), "-o", str(output_bin_path), "-d", dump_ranges], - cwd=simulator_dir, check=True - ) - - -def parse_pim_simulator_outputs(output_bin_path, outputs_descriptor): - raw = output_bin_path.read_bytes() - arrays = [] - offset = 0 - for _, _, dtype_code, shape in outputs_descriptor: - dtype = np.dtype(_ONNX_TO_NP[dtype_code]) - count = int(np.prod(shape)) - array = np.frombuffer(raw, dtype=dtype, count=count, offset=offset).reshape(shape) - offset += count * dtype.itemsize - arrays.append(array) - return arrays - - -def validate_outputs(sim_arrays, runner_out_dir, outputs_descriptor, threshold=1e-3): - all_passed = True - for sim_array, (oi, name, _, shape) in zip(sim_arrays, outputs_descriptor): - csv_name = f"output{oi}_{name}.csv" - runner_array = np.loadtxt(runner_out_dir / csv_name, delimiter=',', dtype=np.float32).reshape(shape) - max_diff = float(np.max(np.abs(sim_array.astype(np.float64) - runner_array.astype(np.float64)))) - passed = max_diff <= threshold - status = Fore.GREEN + "[PASS]" if passed else Fore.RED + "[FAIL]" - print(f" {name}: max diff = {max_diff:.6e} {status}" + Style.RESET_ALL) - if not passed: - all_passed = False - return all_passed - - -if __name__ == '__main__': - ap = argparse.ArgumentParser() - ap.add_argument("--network-onnx", required=True) - ap.add_argument("--raptor-path", required=True) - ap.add_argument("--onnx-include-dir", required=True) +def main(): + ap = argparse.ArgumentParser(description="Validate all ONNX operations under the operations/ directory.") + ap.add_argument("--raptor-path", required=True, help="Path to the Raptor compiler binary.") + ap.add_argument("--onnx-include-dir", required=True, help="Path to OnnxMlirRuntime include directory.") + ap.add_argument("--operations-dir", default=None, help="Root of the operations tree (default: operations).") + ap.add_argument("--simulator-dir", default=None, + help="Path to pim-simulator crate root (default: auto-detected relative to script).") + ap.add_argument("--threshold", type=float, default=1e-3, help="Max allowed diff per output element.") + ap.add_argument("--crossbar-size", type=int, default=64) + ap.add_argument("--crossbar-count", type=int, default=8) a = ap.parse_args() - network_onnx_path = Path(a.network_onnx).absolute() - raptor_path = a.raptor_path + script_dir = Path(__file__).parent.resolve() + operations_dir = Path(a.operations_dir) if a.operations_dir else script_dir / "operations" + simulator_dir = Path(a.simulator_dir) if a.simulator_dir else ( + script_dir / ".." / "backend-simulators" / "pim" / "pim-simulator" + ) - workspace_dir = network_onnx_path.parent - raptor_dir = workspace_dir / "raptor" - runner_dir = workspace_dir / "runner" - runner_build_dir = runner_dir / "build" - Path.mkdir(raptor_dir, exist_ok=True) - Path.mkdir(runner_build_dir, parents=True, exist_ok=True) + if not operations_dir.is_dir(): + print(Fore.RED + f"Operations directory not found: {operations_dir}" + Style.RESET_ALL) + sys.exit(1) - print(Style.BRIGHT + "\nCompiling the onnx network:" + Style.RESET_ALL) - network_so_path, network_mlir_path = compile_onnx_network(network_onnx_path, raptor_path, raptor_dir, runner_dir) + onnx_files = discover_onnx_files(operations_dir) + if not onnx_files: + print(Fore.YELLOW + f"No .onnx files found under {operations_dir}" + Style.RESET_ALL) + sys.exit(1) - print(Style.BRIGHT + "\nGenerating and building the runner:" + Style.RESET_ALL) - gen_network_runner(network_onnx_path, network_so_path, a.onnx_include_dir, out=runner_dir / "runner.c") - runner_path = build_onnx_runner(runner_dir, runner_build_dir) + print(Style.BRIGHT + f"Found {len(onnx_files)} ONNX file(s) to validate.\n" + Style.RESET_ALL) - print(Style.BRIGHT + "\nGenerating random inputs:" + Style.RESET_ALL) - inputs_descriptor, outputs_descriptor = onnx_io(network_onnx_path) - inputs_list, inputs_dict = gen_random_inputs(inputs_descriptor) - flags, _files = save_inputs_to_files(network_onnx_path, inputs_list, out_dir=workspace_dir / "inputs") + results = {} # relative_path -> passed + for onnx_path in onnx_files: + rel = onnx_path.relative_to(operations_dir) + header = f"{'=' * 60}\n Validating: {rel}\n{'=' * 60}" + print(Style.BRIGHT + Fore.CYAN + header + Style.RESET_ALL) - print(Style.BRIGHT + "\nRunning inference with the runner:" + Style.RESET_ALL) - out_dir = workspace_dir / "outputs" - Path.mkdir(out_dir, exist_ok=True) - run_cmd = [runner_path, *flags] - run_cmd += ["--save-csv-dir", f"{out_dir}"] - subprocess.run(run_cmd, cwd=runner_build_dir, check=True) + try: + passed = validate_network( + onnx_path, a.raptor_path, a.onnx_include_dir, simulator_dir, + crossbar_size=a.crossbar_size, crossbar_count=a.crossbar_count, + threshold=a.threshold, + ) + except Exception as e: + print(Fore.RED + f" ERROR: {e}" + Style.RESET_ALL) + passed = False - # Configuration parameters. - matrix_height = matrix_width = 512 - crossbar_size = 64 - crossbar_count = 8 + results[str(rel)] = passed - print(Style.BRIGHT + "Compiling for PIM with Raptor:" + Style.RESET_ALL) - compile_with_raptor(network_mlir_path, raptor_path, crossbar_size, crossbar_count) + # Summary + n_passed = sum(results.values()) + n_total = len(results) + print("\n" + Style.BRIGHT + "=" * 60) + print(" Summary") + print("=" * 60 + Style.RESET_ALL) + for rel, passed in results.items(): + status = Fore.GREEN + "PASS" if passed else Fore.RED + "FAIL" + print(f" {rel}: {status}" + Style.RESET_ALL) + print(Style.BRIGHT + f"\n {n_passed}/{n_total} passed." + Style.RESET_ALL) - print(Style.BRIGHT + "\nRunning PIM simulation:" + Style.RESET_ALL) - pim_dir = raptor_dir / "pim" - write_inputs_to_memory_bin(pim_dir / "memory.bin", pim_dir / "config.json", inputs_list) - simulator_dir = Path(__file__).parent.resolve() / ".." / "backend-simulators" / "pim" / "pim-simulator" - simulation_dir = workspace_dir / "simulation" - Path.mkdir(simulation_dir, exist_ok=True) - dump_ranges = build_dump_ranges(pim_dir / "config.json", outputs_descriptor) - output_bin_path = simulation_dir / "out.bin" - run_pim_simulator(simulator_dir, pim_dir, output_bin_path, dump_ranges) + sys.exit(0 if n_passed == n_total else 1) - print(Style.BRIGHT + "\nValidating the results:" + Style.RESET_ALL) - sim_arrays = parse_pim_simulator_outputs(output_bin_path, outputs_descriptor) - validate_outputs(sim_arrays, out_dir, outputs_descriptor) + +if __name__ == "__main__": + main() diff --git a/validation/validate_one.py b/validation/validate_one.py new file mode 100644 index 0000000..9c89940 --- /dev/null +++ b/validation/validate_one.py @@ -0,0 +1,137 @@ +import argparse +import json +import numpy as np +import subprocess +from pathlib import Path +from colorama import Style, Fore +from onnx_utils import gen_random_inputs, save_inputs_to_files, onnx_io, write_inputs_to_memory_bin, _ONNX_TO_NP +from raptor import compile_with_raptor +from gen_network_runner import gen_network_runner + + +def compile_onnx_network(network_onnx_path, raptor_path, raptor_dir, runner_dir): + subprocess.run([raptor_path, network_onnx_path, "--EmitONNXIR"], check=True) + subprocess.run([raptor_path, network_onnx_path], check=True) + parent = network_onnx_path.parent + stem = network_onnx_path.stem + so_path = parent / f"{stem}.so" + mlir_path = parent / f"{stem}.onnx.mlir" + tmp_path = parent / f"{stem}.tmp" + moved_so = runner_dir / so_path.name + moved_mlir = raptor_dir / mlir_path.name + so_path.rename(moved_so) + mlir_path.rename(moved_mlir) + tmp_path.unlink(missing_ok=True) + return moved_so, moved_mlir + + +def build_onnx_runner(source_dir, build_dir): + subprocess.run(["cmake", source_dir], cwd=build_dir, check=True) + subprocess.run(["cmake", "--build", ".", "-j"], cwd=build_dir, check=True) + return build_dir / "runner" + + +def build_dump_ranges(config_path, outputs_descriptor): + with open(config_path) as f: + output_addresses = json.load(f)["outputs_addresses"] + ranges = [] + for addr, (_, _, dtype_code, shape) in zip(output_addresses, outputs_descriptor): + byte_size = int(np.prod(shape)) * np.dtype(_ONNX_TO_NP[dtype_code]).itemsize + ranges.append(f"{addr},{byte_size}") + return ",".join(ranges) + + +def run_pim_simulator(simulator_dir, pim_dir, output_bin_path, dump_ranges): + subprocess.run( + ["cargo", "run", "--release", "--package", "pim-simulator", "--bin", "pim-simulator", "--", + "-f", str(pim_dir), "-o", str(output_bin_path), "-d", dump_ranges], + cwd=simulator_dir, check=True + ) + + +def parse_pim_simulator_outputs(output_bin_path, outputs_descriptor): + raw = output_bin_path.read_bytes() + arrays = [] + offset = 0 + for _, _, dtype_code, shape in outputs_descriptor: + dtype = np.dtype(_ONNX_TO_NP[dtype_code]) + count = int(np.prod(shape)) + array = np.frombuffer(raw, dtype=dtype, count=count, offset=offset).reshape(shape) + offset += count * dtype.itemsize + arrays.append(array) + return arrays + + +def validate_outputs(sim_arrays, runner_out_dir, outputs_descriptor, threshold=1e-3): + all_passed = True + for sim_array, (oi, name, _, shape) in zip(sim_arrays, outputs_descriptor): + csv_name = f"output{oi}_{name}.csv" + runner_array = np.loadtxt(runner_out_dir / csv_name, delimiter=',', dtype=np.float32).reshape(shape) + max_diff = float(np.max(np.abs(sim_array.astype(np.float64) - runner_array.astype(np.float64)))) + passed = max_diff <= threshold + status = Fore.GREEN + "[PASS]" if passed else Fore.RED + "[FAIL]" + print(f" {name}: max diff = {max_diff:.6e} {status}" + Style.RESET_ALL) + if not passed: + all_passed = False + return all_passed + + +def validate_network(network_onnx_path, raptor_path, onnx_include_dir, + simulator_dir, crossbar_size=64, crossbar_count=8, threshold=1e-3): + network_onnx_path = Path(network_onnx_path).absolute() + workspace_dir = network_onnx_path.parent + raptor_dir = workspace_dir / "raptor" + runner_dir = workspace_dir / "runner" + runner_build_dir = runner_dir / "build" + Path.mkdir(raptor_dir, exist_ok=True) + Path.mkdir(runner_build_dir, parents=True, exist_ok=True) + + print(Style.BRIGHT + "\nCompiling the onnx network:" + Style.RESET_ALL) + network_so_path, network_mlir_path = compile_onnx_network(network_onnx_path, raptor_path, raptor_dir, runner_dir) + + print(Style.BRIGHT + "\nGenerating and building the runner:" + Style.RESET_ALL) + gen_network_runner(network_onnx_path, network_so_path, onnx_include_dir, out=runner_dir / "runner.c") + runner_path = build_onnx_runner(runner_dir, runner_build_dir) + + print(Style.BRIGHT + "\nGenerating random inputs:" + Style.RESET_ALL) + inputs_descriptor, outputs_descriptor = onnx_io(network_onnx_path) + inputs_list, inputs_dict = gen_random_inputs(inputs_descriptor) + flags, _files = save_inputs_to_files(network_onnx_path, inputs_list, out_dir=workspace_dir / "inputs") + + print(Style.BRIGHT + "\nRunning inference with the runner:" + Style.RESET_ALL) + out_dir = workspace_dir / "outputs" + Path.mkdir(out_dir, exist_ok=True) + run_cmd = [runner_path, *flags] + run_cmd += ["--save-csv-dir", f"{out_dir}"] + subprocess.run(run_cmd, cwd=runner_build_dir, check=True) + + print(Style.BRIGHT + "\nCompiling for PIM with Raptor:" + Style.RESET_ALL) + compile_with_raptor(network_mlir_path, raptor_path, crossbar_size, crossbar_count) + + print(Style.BRIGHT + "\nRunning PIM simulation:" + Style.RESET_ALL) + pim_dir = raptor_dir / "pim" + write_inputs_to_memory_bin(pim_dir / "memory.bin", pim_dir / "config.json", inputs_list) + simulation_dir = workspace_dir / "simulation" + Path.mkdir(simulation_dir, exist_ok=True) + dump_ranges = build_dump_ranges(pim_dir / "config.json", outputs_descriptor) + output_bin_path = simulation_dir / "out.bin" + run_pim_simulator(simulator_dir, pim_dir, output_bin_path, dump_ranges) + + print(Style.BRIGHT + "\nValidating the results:" + Style.RESET_ALL) + sim_arrays = parse_pim_simulator_outputs(output_bin_path, outputs_descriptor) + return validate_outputs(sim_arrays, out_dir, outputs_descriptor, threshold) + + +if __name__ == '__main__': + ap = argparse.ArgumentParser() + ap.add_argument("--network-onnx", required=True) + ap.add_argument("--raptor-path", required=True) + ap.add_argument("--onnx-include-dir", required=True) + a = ap.parse_args() + + simulator_dir = Path(__file__).parent.resolve() / ".." / "backend-simulators" / "pim" / "pim-simulator" + + passed = validate_network( + a.network_onnx, a.raptor_path, a.onnx_include_dir, simulator_dir + ) + raise SystemExit(0 if passed else 1)