From 0cc83f7c97e20992a9b4c3580e698064d276bc9c Mon Sep 17 00:00:00 2001 From: NiccoloN Date: Thu, 6 Aug 2026 14:52:55 +0200 Subject: [PATCH] restore unwanted changes --- validation/.gitignore | 2 - .../googlenet/googlenet-12.onnx | Bin 28021885 -> 28021836 bytes .../networks/pimcomp_models/results.csv | 6 +- validation/tools/analyze_yolo11n_attention.py | 182 ------------------ 4 files changed, 4 insertions(+), 186 deletions(-) delete mode 100644 validation/tools/analyze_yolo11n_attention.py diff --git a/validation/.gitignore b/validation/.gitignore index f24a90f..0d62ec7 100644 --- a/validation/.gitignore +++ b/validation/.gitignore @@ -20,5 +20,3 @@ networks/**/*.csv !networks/full_net/validation_results.csv !networks/pimcomp_models/validation_results.csv !networks/pimcomp_models/results.csv -!networks/pimcomp_models/validation_results.csv -!operations/validation_results.csv diff --git a/validation/networks/pimcomp_models/googlenet/googlenet-12.onnx b/validation/networks/pimcomp_models/googlenet/googlenet-12.onnx index 97a4db3480c969c153e3549f588b2e1860852633..1865acc0a19390cd1f562b32f4f87751b873fdae 100644 GIT binary patch delta 1489 zcmWm6XTS&q00!YcgzU4Eor;9)k+MP=84+ z=jYwuJFhRjFDPEDe9383r_O9NX4LrcaP(ni`!J2^_)%h7U-bdZkHNjggxIaaz#H|Z|N$??)ddP*

g%2FAvCr@{l|%kI19)m^>~| z$dmGv?31VE8F^Noljr3Hc~M@Hm*o|CRbG?VzzLKxy8~IkglkepR`B8q7pXC?%ReqD-5onOq7js@&BWIREUaEDJn;ms2T@GwKyoMM~$c%wW4;^iMnxc z)Qdyn&^RpWM}spaXPC>lqTI5L_>vuGYIqGhy-*3l-~M!PsF+Q-pxOmv8j(J4Ad zmpC@MMz`o5$Hno{BYH-!=p83SpXeL?qJNwi17cvD6oX=L42hFtXbg+tF(O9BDKRQW z$Cwx!<6?YFh>0;NPL0ViB~FW}F)dDy=`kbDh%@7?I6KaXnQ?BM7w5+Xaba8(7ssra z9dlxC%nM_FEQp1%C@zV`u_TtpvRED~;?lS*E{~OQMO+!HVs)&EwQ*Ifi}kS~HpZsd j99!b**c#hnd+dmvaZT(h*vsyMmMe>PE>dE@a;5$Qj6=l; delta 1526 zcmWm6XMhj_07r30D0EvOGb?v*`e!4vylSXQFf9FQc)^NWhs;@QY2NS znpBq>Qd4S4lG;*7>PkJSFAb!jG?JZV7ila_q^UHM=F&oTmEB}_X(_E_57|>%%U;q( z_LjD?kF=A0rM+~Jjss~ji?$-&Z1x=Rn~DZQk(^pQj4P&rHvmm}mz zIZBR}zS2+n%Q14S43L2`NCwMsa=Z+Y6XZlWNluobGE9o)6e*EYWw?xxkupj~%NRLL z#>zMuFB7Cx%4DLPE@w!&oGFuJvP_Ywa+XY!>2kK5BQs>C%#ztMN6wYGqRf-?F1cIok$YvU+$Zl#k?N z`9waI&*XFYLcWx*=-*mg{T;nqH+{Ql_-j;Q7x)Rji?#5;{QkOs1tRgUeu2U(J>&aq20jwaDGnnm+y z5xd52v3saD9 zjdig;E{`kX%D5`7jty~5Y>Z8@xnL_>3OX%sI<{R=-QtO5W#c=w>oT=Vae4dV2_@y@ cird$2P*kt5%DM-aZ7HZzsaJ)f9coqk4=vWv8UO$Q diff --git a/validation/networks/pimcomp_models/results.csv b/validation/networks/pimcomp_models/results.csv index 69c4816..49345f5 100644 --- a/validation/networks/pimcomp_models/results.csv +++ b/validation/networks/pimcomp_models/results.csv @@ -1,3 +1,5 @@ model,raptor_latency_ms,pimcomp_latency_ms,raptor_energy_pj,pimcomp_energy_pj,faster_compiler,speedup -vgg8,1.521060,7.985074,486309111.040001,1597904071.120000,raptor,5.25 -resnet18,33.552733,58.853613,9702508727.119982,13983148468.119974,raptor,1.75 +vgg8,1.465778,7.985074,477298145.040001,1597904071.120000,raptor,5.45 +resnet18,28.099952,58.853733,8781611766.119984,13983168748.119974,raptor,2.09 +resnet34,45.781486,91.607980,14962940227.679951,22722922369.680016,raptor,2.00 +googlenet,13.371204,62.923463,6117835798.919991,14547526780.240000,raptor,4.71 diff --git a/validation/tools/analyze_yolo11n_attention.py b/validation/tools/analyze_yolo11n_attention.py deleted file mode 100644 index 8453e4f..0000000 --- a/validation/tools/analyze_yolo11n_attention.py +++ /dev/null @@ -1,182 +0,0 @@ -#!/usr/bin/env python3 -"""Save exact attention-tap arrays and quantify the MatMul error sources.""" - -import argparse -import hashlib -import json -import sys -from pathlib import Path - -import numpy as np -import onnx -from onnx import numpy_helper - -REPO_ROOT = Path(__file__).resolve().parents[2] -sys.path.insert(0, str(REPO_ROOT / "validation")) - -from raptor_validation.onnx_utils import onnx_io # noqa: E402 -from raptor_validation.validate_one import ( # noqa: E402 - parse_pim_simulator_outputs, - sanitize_output_name, -) - - -TAP_NAMES = { - "v": "/model.10/m/m.0/attn/Split_output_2", - "raw": "/model.10/m/m.0/attn/MatMul_output_0", - "scaled": "/model.10/m/m.0/attn/Mul_output_0", - "rhs": "/model.10/m/m.0/attn/Transpose_1_output_0", - "c": "/model.10/m/m.0/attn/MatMul_1_output_0", -} -ABSOLUTE_TOLERANCE = 1e-3 -RELATIVE_TOLERANCE = 1e-5 - - -def sha256(path): - digest = hashlib.sha256() - with Path(path).open("rb") as stream: - for block in iter(lambda: stream.read(1 << 20), b""): - digest.update(block) - return digest.hexdigest() - - -def metric(actual, expected): - difference = np.abs(actual.astype(np.float64) - expected.astype(np.float64)) - allowed = ABSOLUTE_TOLERANCE + RELATIVE_TOLERANCE * np.abs(expected.astype(np.float64)) - return { - "max_abs": float(np.max(difference)), - "mean_abs": float(np.mean(difference)), - "rms": float(np.sqrt(np.mean(np.square(difference)))), - "elements_over_validator_limit": int(np.count_nonzero(difference > allowed)), - } - - -def f32_matmul(lhs, rhs): - return np.matmul(lhs.astype(np.float32), rhs.astype(np.float32)).astype(np.float32) - - -def f64_matmul(lhs, rhs): - return np.matmul(lhs.astype(np.float64), rhs.astype(np.float64)).astype(np.float64) - - -def load_constant(model, output_name): - for initializer in model.graph.initializer: - if initializer.name == output_name: - return float(numpy_helper.to_array(initializer).reshape(-1)[0]) - for node in model.graph.node: - if output_name not in node.output: - continue - for attribute in node.attribute: - if attribute.name == "value" and attribute.HasField("t"): - return float(numpy_helper.to_array(attribute.t).reshape(-1)[0]) - raise ValueError(f"could not find ONNX Constant producing {output_name}") - - -def load_arrays(workspace, model_path): - model = onnx.load(model_path) - descriptors = onnx_io(model_path) - output_descriptors = {name: (index, dtype, shape) for index, name, dtype, shape in descriptors[1]} - missing = sorted(set(TAP_NAMES.values()) - set(output_descriptors)) - if missing: - raise ValueError("tap model is missing outputs: " + ", ".join(missing)) - - sim_arrays = parse_pim_simulator_outputs( - workspace / "simulation" / "out.bin", descriptors[1] - ) - reference = {} - simulated = {} - input_files = {} - for key, name in TAP_NAMES.items(): - index, _dtype, shape = output_descriptors[name] - csv_path = workspace / "outputs" / f"output{index}_{sanitize_output_name(name)}.csv" - reference[key] = np.loadtxt(csv_path, delimiter=",", dtype=np.float32).reshape(shape) - simulated[key] = np.asarray(sim_arrays[index], dtype=np.float32).reshape(shape) - input_files[key] = csv_path - return reference, simulated, input_files - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--workspace", type=Path, required=True, - help="validator workspace containing inputs, outputs, and simulation") - parser.add_argument("--model", type=Path, required=True, help="five-output ONNX tap model") - parser.add_argument("--output-dir", type=Path, required=True, - help="directory for arrays.npz, metadata.json, and decomposition.json") - args = parser.parse_args() - args.output_dir.mkdir(parents=True, exist_ok=True) - - model = onnx.load(args.model) - reference, simulated, source_files = load_arrays(args.workspace, args.model) - scale = np.float32(load_constant(model, "/model.10/m/m.0/attn/Constant_1_output_0")) - ref_v, ref_raw, ref_scaled, ref_rhs, ref_c = (reference[key] for key in ("v", "raw", "scaled", "rhs", "c")) - sim_v, sim_raw, sim_scaled, sim_rhs, sim_c = (simulated[key] for key in ("v", "raw", "scaled", "rhs", "c")) - - ref_score_transpose = np.swapaxes(ref_scaled, -1, -2) - sim_score_transpose = np.swapaxes(sim_scaled, -1, -2) - ref_ss_f32 = f32_matmul(ref_v, ref_rhs) - sim_ss_f32 = f32_matmul(sim_v, sim_rhs) - ref_ss_f64 = f64_matmul(ref_v, ref_rhs) - sim_ss_f64 = f64_matmul(sim_v, sim_rhs) - ref_split_f32 = (f32_matmul(ref_v, np.swapaxes(ref_raw, -1, -2)) * scale).astype(np.float32) - sim_split_f32 = (f32_matmul(sim_v, np.swapaxes(sim_raw, -1, -2)) * scale).astype(np.float32) - ref_split_f64 = f64_matmul(ref_v, np.swapaxes(ref_raw, -1, -2)) * np.float64(scale) - sim_split_f64 = f64_matmul(sim_v, np.swapaxes(sim_raw, -1, -2)) * np.float64(scale) - - arrays = { - **{f"ref_{key}": value for key, value in reference.items()}, - **{f"sim_{key}": value for key, value in simulated.items()}, - "ref_ss_f32": ref_ss_f32, - "sim_ss_f32": sim_ss_f32, - "ref_ss_f64": ref_ss_f64, - "sim_ss_f64": sim_ss_f64, - "ref_split_f32": ref_split_f32, - "sim_split_f32": sim_split_f32, - "ref_split_f64": ref_split_f64, - "sim_split_f64": sim_split_f64, - } - arrays_path = args.output_dir / "arrays.npz" - np.savez_compressed(arrays_path, **arrays) - - metrics = { - "validator_policy": { - "absolute_tolerance": ABSOLUTE_TOLERANCE, - "relative_tolerance": RELATIVE_TOLERANCE, - }, - "scale": float(scale), - "shape": list(ref_c.shape), - "tap_differences": {key: metric(simulated[key], reference[key]) for key in TAP_NAMES}, - "rhs_transpose_consistency": metric(ref_rhs, ref_score_transpose), - "sim_rhs_transpose_consistency": metric(sim_rhs, sim_score_transpose), - "c_sim_vs_ss_f32": metric(sim_c, ref_ss_f32), - "c_ref_vs_ss_f32": metric(ref_c, ref_ss_f32), - "c_sim_vs_simulated_inputs_ss_f32": metric(sim_c, sim_ss_f32), - "v_drift_only": metric(f32_matmul(sim_v, ref_rhs), ref_ss_f32), - "rhs_drift_only": metric(f32_matmul(ref_v, sim_rhs), ref_ss_f32), - "joint_input_drift": metric(sim_ss_f32, ref_ss_f32), - "scale_reassociation_reference": metric(ref_split_f32, ref_ss_f32), - "scale_reassociation_simulated": metric(sim_split_f32, sim_ss_f32), - "reference_accumulation_f32_vs_f64": metric(ref_ss_f32, ref_ss_f64), - "simulated_accumulation_f32_vs_f64": metric(sim_ss_f32, sim_ss_f64), - "split_accumulation_reference_f32_vs_f64": metric(ref_split_f32, ref_split_f64), - "split_accumulation_simulated_f32_vs_f64": metric(sim_split_f32, sim_split_f64), - } - decomposition_path = args.output_dir / "decomposition.json" - decomposition_path.write_text(json.dumps(metrics, indent=2) + "\n", encoding="utf-8") - - metadata = { - "model": str(args.model), - "model_sha256": sha256(args.model), - "workspace": str(args.workspace), - "arrays_sha256": sha256(arrays_path), - "source_sha256": {key: sha256(path) for key, path in source_files.items()}, - "simulator_output_sha256": sha256(args.workspace / "simulation" / "out.bin"), - "input_sha256": sha256(args.workspace / "inputs" / "in0.csv"), - "outputs": TAP_NAMES, - "arrays": {key: {"dtype": str(value.dtype), "shape": list(value.shape)} for key, value in arrays.items()}, - } - (args.output_dir / "metadata.json").write_text(json.dumps(metadata, indent=2) + "\n", encoding="utf-8") - print(json.dumps(metrics, indent=2)) - - -if __name__ == "__main__": - main()