From 5415e95528c56ce49d33762187b139c74d582e85 Mon Sep 17 00:00:00 2001 From: NiccoloN Date: Mon, 27 Jul 2026 11:32:45 +0200 Subject: [PATCH] parallel validation jobs update readme --- AGENTS.md | 1 + README.md | 106 +------------------ validation/README.md | 225 +++++++++++++++++++++++++++++++++++++++++ validation/validate.py | 150 ++++++++++++++++++++------- 4 files changed, 346 insertions(+), 136 deletions(-) create mode 100644 validation/README.md diff --git a/AGENTS.md b/AGENTS.md index 2572ece..7d2047d 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -15,6 +15,7 @@ Before modifying the relevant subsystem, read: * The debug build is very slow, so use it only on small fast tests such as operation validations, not on network validations * Always prepend rtk to shell commands if missing and if rtk is available * Always use the repository Python virtual environment (`.venv/bin/python` and its bundled tools) for Python commands +* Run the complete operations validation (`--operations-dir validation/operations`) outside the sandbox so parallel worker processes can start. This does not apply to one-at-a-time network validation # Core engineering philosophy diff --git a/README.md b/README.md index 137c068..5cad246 100644 --- a/README.md +++ b/README.md @@ -154,108 +154,10 @@ This writes PIM artifacts under `/tmp/raptor/pim/`. ## Validation -Functional validation lives in `validation/`. It compiles ONNX models, builds a -native ONNX-MLIR reference runner, generates random inputs, runs Raptor, runs -the Rust functional simulator, and compares outputs. - -Python dependencies used by the validation scripts are `numpy`, `onnx`, and -`colorama`. The simulator requires the Rust toolchain. - -Per-operation validation from the repository root: - -```bash -.venv/bin/python validation/validate.py \ - --raptor-path build_release/Release/bin/onnx-mlir \ - --onnx-include-dir onnx-mlir/include \ - --operations-dir validation/operations \ - --verbose \ - --raptor-extra-arg=--pim-detect-communication-deadlock \ - --raptor-extra-arg=--pim-export-spatial-dataflow=none -``` - -Validate one network or a subset by pointing `--operations-dir` at any directory -containing `.onnx` files: - -```bash -.venv/bin/python validation/validate.py \ - --raptor-path build_release/Release/bin/onnx-mlir \ - --onnx-include-dir onnx-mlir/include \ - --operations-dir validation/networks/yolo11n/depth_04 \ - --verbose \ - --raptor-extra-arg=--pim-detect-communication-deadlock \ - --raptor-extra-arg=--pim-export-spatial-dataflow=none -``` - -Useful validation options: -- `--simulator-dir ` - override the auto-detected functional - `backend-simulators/pim/pim-simulator` path. -- `--pimcomp-config ` - select the checked-in - non-functional simulation profile. Arch-A is the default; the hardware - argument defaults match it (`168` cores, `96` crossbars/core, `128x128` - crossbars). - Selecting another profile also requires its matching hardware arguments. -- `--non-functional-simulator-build-dir ` - override the auto-detected - non-functional simulator build directory. -- `--skip-non-functional-simulation` - skip non-functional simulation. If - explicit core/crossbar arguments do not match the selected profile, validation - reports the mismatch and skips only non-functional simulation. -- `--threshold ` - maximum allowed per-element output difference. -- `--seed ` - RNG seed for generated inputs. -- `--command-timeout-seconds ` - timeout for compiler, runner, and - simulator subprocesses. -- `--verbose` - print subprocess logs and average PIM pass timings. -- `--clean` - remove generated validation artifacts and exit. - -Each validation run writes artifacts in the model workspace, for example under -`validation/operations/gemm/small/`: -- `inputs/` - generated input CSV files. -- `outputs/` - native ONNX-MLIR reference outputs. -- `raptor/` - compiler artifacts, including `*.onnx.mlir`, dialect dumps under - `dialects/`, reports under `reports/`, and final PIM artifacts under `pim/`. -- `runner/` - generated reference runner source, build tree, and shared library. -- `simulation/out.bin` - raw functional simulation output used for comparison. - -By default, validation also runs non-functional simulation. Per-model `Latency` -is shown in the result table alongside `Power`, with measured, failed, and -skipped counts plus total latency in the summary. Use these validation results -for performance comparisons; the simulation backend does not need to be invoked -separately. The selected pre-generated configs and meshes live under -`validation/pimsim_configs/pimcomp/`. - -The compiler currently dumps dialect snapshots such as `spatial0.mlir`, -`spatial1_graph.mlir`, `spatial2_trivial_merged.mlir`, -`spatial3_scheduled_no_comm.mlir`, `spatial4_scheduled.mlir`, `pim0.mlir`, `pim1_buff.mlir`, -`pim2_folded.mlir`, and `pim3_memory_planned.mlir` when an output directory is -available. - -To rerun the functional simulator manually with tracing after validation has -produced a `raptor/pim/` directory: - -```bash -cd backend-simulators/pim/pim-simulator -cargo run --no-default-features --features tracing --release \ - --package pim-simulator --bin pim-simulator -- \ - -f /path/to/workspace/raptor/pim \ - -o /path/to/workspace/simulation/out.bin \ - -d ,,,,... -``` - -With `--features tracing`, the simulator writes per-core traces as -`TraceCore0`, `TraceCore1`, ... next to `out.bin`. The validator normally -computes the `-d` ranges from `raptor/pim/config.json` and model output shapes. - -Available validation networks under `validation/networks/`: `vgg16`, -`yolo11n`, `yolo11nv2`. - -Available operation suites under `validation/operations/`: `add`, `concat`, -`conv`, `div`, `gather`, `gemm`, `gemv`, `matmul`, `mul`, `pool`, -`reduce_mean`, `relu`, `reshape`, `resize`, `sigmoid`, `softmax`, `split`. - -Generated operation tests can be regenerated with: - -```bash -.venv/bin/python validation/operations/gen_tests.py -``` +Functional validation compiles ONNX models, compares native ONNX-MLIR and PIM +simulator outputs, and optionally reports latency and power. See +[`validation/README.md`](validation/README.md) for prerequisites, usage, +options, artifacts, and results. ## Build diff --git a/validation/README.md b/validation/README.md new file mode 100644 index 0000000..2cc8079 --- /dev/null +++ b/validation/README.md @@ -0,0 +1,225 @@ +# Raptor Validation + +`validate.py` validates every ONNX model below a selected directory. For each +model it can: + +1. compile an ONNX-MLIR reference library and runner; +2. generate deterministic random inputs; +3. compile PIM artifacts with Raptor; +4. run the reference implementation and functional PIM simulator; +5. compare their outputs; +6. run `pimsim-nn` to report latency and power. + +Run the script from the repository root with the repository Python environment. + +## Prerequisites + +- A built Raptor compiler, normally + `build_release/Release/bin/onnx-mlir`. +- ONNX-MLIR runtime headers, normally `onnx-mlir/include`. +- The `numpy`, `onnx`, and `colorama` packages installed in `.venv`. +- The Rust toolchain used by the functional simulator. +- The Rust functional simulator under + `backend-simulators/pim/pim-simulator`, unless overridden. +- A built `pimsim-nn` under + `backend-simulators/pim/pimsim-nn/build`, unless non-functional simulation is + skipped or its path is overridden. + +## Basic usage + +Validate the complete operation suite: + +```bash +.venv/bin/python validation/validate.py \ + --raptor-path build_release/Release/bin/onnx-mlir \ + --onnx-include-dir onnx-mlir/include \ + --operations-dir validation/operations \ + --verbose \ + --raptor-extra-arg=--pim-detect-communication-deadlock \ + --raptor-extra-arg=--pim-export-spatial-dataflow=none +``` + +Validate one operation category or case: + +```bash +.venv/bin/python validation/validate.py \ + --raptor-path build_release/Release/bin/onnx-mlir \ + --onnx-include-dir onnx-mlir/include \ + --operations-dir validation/operations/gemm/small +``` + +Validate a network or network slice: + +```bash +.venv/bin/python validation/validate.py \ + --raptor-path build_release/Release/bin/onnx-mlir \ + --onnx-include-dir onnx-mlir/include \ + --operations-dir validation/networks/yolo11n/depth_04 +``` + +`--operations-dir` may point to any directory tree containing `.onnx` files. +The script discovers them recursively. + +## Validation modes + +The default mode performs the complete workflow. + +Use `--compile-only` to build the reference runner and PIM artifacts without +executing either implementation: + +```bash +.venv/bin/python validation/validate.py \ + --raptor-path build_release/Release/bin/onnx-mlir \ + --onnx-include-dir onnx-mlir/include \ + --operations-dir validation/operations/gemm/small \ + --compile-only +``` + +Use `--run-only` to reuse those artifacts and perform input generation, +reference execution, simulation, and comparison: + +```bash +.venv/bin/python validation/validate.py \ + --raptor-path build_release/Release/bin/onnx-mlir \ + --onnx-include-dir onnx-mlir/include \ + --operations-dir validation/operations/gemm/small \ + --run-only +``` + +`--compile-only` and `--run-only` are mutually exclusive. Run-only mode fails +with a diagnostic if its required compiled artifacts are missing. + +## Parallel execution and output + +Models run in parallel using all available CPUs by default. Set the worker +count with `-j` or `--jobs`: + +```bash +.venv/bin/python validation/validate.py \ + --raptor-path build_release/Release/bin/onnx-mlir \ + --onnx-include-dir onnx-mlir/include \ + --operations-dir validation/operations \ + --jobs 8 +``` + +Each model's output is buffered as one readable block. Independent workspace +jobs print their blocks when they finish, so their order may differ from the +final table. The final result table is sorted by model path. + +Models in the same directory run sequentially because they share generated +workspace paths. + +## Options + +| Option | Description | +|---|---| +| `-h`, `--help` | Print command help and exit. | +| `--raptor-path PATH` | Raptor compiler binary. Required unless `--clean` is used. | +| `--onnx-include-dir PATH` | ONNX-MLIR runtime include directory. Required unless `--clean` is used. | +| `--operations-dir PATH` | Directory tree containing models. Defaults to `validation/operations`. | +| `--simulator-dir PATH` | Functional `pim-simulator` crate directory. Defaults to the in-tree simulator. | +| `--non-functional-simulator-build-dir PATH` | `pimsim-nn` build directory. Defaults to the in-tree build. | +| `--pimcomp-config {arch-a,arch-b,arch-c}` | Non-functional hardware/timing profile. Defaults to `arch-a`. | +| `--skip-non-functional-simulation` | Skip `pimsim-nn` latency and power measurement. | +| `--threshold FLOAT` | Absolute output-comparison tolerance. Defaults to `1e-3`. | +| `--relative-threshold FLOAT` | Relative output-comparison tolerance. Defaults to `1e-5`. | +| `--seed INT` | Seed for generated inputs. Defaults to `0`. | +| `--crossbar-size INT` | Crossbar dimensions passed to Raptor. Defaults to the Arch-A value, `128`. | +| `--crossbar-count INT` | Crossbars per core passed to Raptor. Defaults to the Arch-A value, `96`. | +| `--core-count INT` | PIM core count passed to Raptor. Defaults to the Arch-A value, `168`. | +| `--raptor-extra-arg=ARG` | Additional Raptor compiler argument. Repeat for multiple arguments. | +| `--command-timeout-seconds FLOAT` | Timeout for each compiler, runner, and simulator subprocess. Defaults to `1000000.0`. | +| `-j INT`, `--jobs INT` | Parallel validation workers. Defaults to all available CPUs and must be at least one. | +| `--clean` | Remove generated validation artifacts and exit. | +| `--compile-only` | Compile reference and PIM artifacts without execution or comparison. | +| `--run-only` | Reuse compiled artifacts and perform execution, simulation, and comparison. | +| `--verbose` | Print passing per-stage and subprocess logs, plus average PIM pass timings. | + +Arguments beginning with `--` that are passed through to Raptor should use the +equals form: + +```bash +--raptor-extra-arg=--pim-detect-communication-deadlock +``` + +## Hardware profiles and non-functional simulation + +The selected PIMCOMP profile must match `--core-count`, `--crossbar-count`, and +`--crossbar-size`. A mismatch disables only non-functional simulation and +prints the incompatible values; functional validation still runs. + +The checked-in profiles are under +`validation/pimsim_configs/pimcomp//latency_config.json`. + +Use `--skip-non-functional-simulation` when latency and power are not required. +The summary reports non-functional results as measured, failed, or skipped. + +## Generated artifacts + +Artifacts are written beside each model: + +| Path | Contents | +|---|---| +| `inputs/` | Generated input CSV files. | +| `outputs/` | ONNX-MLIR reference output CSV files. | +| `raptor/` | Exported MLIR, dialect snapshots, reports, and final `pim/` artifacts. | +| `runner/` | Generated reference runner source, build tree, and shared library. | +| `simulation/out.bin` | Functional simulator output used for comparison. | + +The `raptor/` directory may include `spatial0.mlir`, +`spatial1_graph.mlir`, `spatial2_trivial_merged.mlir`, +`spatial3_scheduled_no_comm.mlir`, `spatial4_scheduled.mlir`, `pim0.mlir`, +`pim1_buff.mlir`, `pim2_folded.mlir`, and `pim3_memory_planned.mlir`. + +Remove these artifacts for every discovered model with: + +```bash +.venv/bin/python validation/validate.py \ + --operations-dir validation/operations/gemm/small \ + --clean +``` + +`--clean` does not require `--raptor-path` or `--onnx-include-dir`. + +## Available suites + +Checked-in validation networks under `validation/networks/` include `vgg16`, +`yolo11n`, and `yolo11nv2`. + +The generated operation inventory is documented in +[`operations/README.md`](operations/README.md). Regenerate its models with: + +```bash +.venv/bin/python validation/operations/gen_tests.py +``` + +## Manual functional simulator tracing + +After validation has produced a `raptor/pim/` directory, rerun the functional +simulator with tracing from its crate directory: + +```bash +cd backend-simulators/pim/pim-simulator +cargo run --no-default-features --features tracing --release \ + --package pim-simulator --bin pim-simulator -- \ + -f /path/to/workspace/raptor/pim \ + -o /path/to/workspace/simulation/out.bin \ + -d ,,,,... +``` + +Tracing writes `TraceCore0`, `TraceCore1`, and so on beside `out.bin`. The +validator normally derives the `-d` address and byte ranges from +`raptor/pim/config.json` and the model output shapes. + +## Results and exit status + +The final table reports functional pass/fail state and non-functional latency +and power. The summary includes pass/fail totals, non-functional simulation +counts, total measured latency, and average PIM pass timings when `--verbose` +is enabled. + +- Exit status `0`: all discovered models passed, or cleanup completed. +- Exit status `1`: validation failed, the model directory was invalid, or no + `.onnx` models were found. +- Exit status `2`: command-line arguments were invalid or required arguments + were missing. diff --git a/validation/validate.py b/validation/validate.py index 44534b5..3442008 100755 --- a/validation/validate.py +++ b/validation/validate.py @@ -1,10 +1,15 @@ #!/usr/bin/env python3 import argparse +import os import signal import subprocess import sys +from concurrent.futures import ProcessPoolExecutor, as_completed +from contextlib import redirect_stderr, redirect_stdout +from itertools import groupby from pathlib import Path +from tempfile import TemporaryDirectory from colorama import Style, Fore from raptor_validation.validate_one import ( MODE_COMPILE_ONLY, @@ -28,6 +33,49 @@ DEFAULT_PIMCOMP_CONFIG = "arch-a" PIMCOMP_CONFIG_CHOICES = ("arch-a", "arch-b", "arch-c") +def run_validation_job(job): + models, options = job + completed = [] + for index, onnx_path, rel, log_path in models: + reporter = ProgressReporter( + options["model_total"], + stages_per_model=len(MODE_STAGE_TITLES[options["mode"]]), + enabled=False, + verbose=options["verbose"], + ) + sys.stdout.flush() + sys.stderr.flush() + saved_stdout = os.dup(1) + saved_stderr = os.dup(2) + try: + with open(log_path, "w", encoding="utf-8", buffering=1) as log: + os.dup2(log.fileno(), 1) + os.dup2(log.fileno(), 2) + with redirect_stdout(log), redirect_stderr(log): + try: + result = validate_network( + onnx_path, + reporter=reporter, + model_index=index, + model_total=options["model_total"], + verbose=options["verbose"], + mode=options["mode"], + **options["validation_kwargs"], + ) + except Exception as exc: + result = ValidationResult(False, pimsim_status=PIMSIM_NOT_RUN) + print_validation_error(reporter, rel, exc) + finally: + reporter.finish() + finally: + os.dup2(saved_stdout, 1) + os.dup2(saved_stderr, 2) + os.close(saved_stdout) + os.close(saved_stderr) + completed.append((str(rel), result, str(log_path))) + return completed + + def format_return_status(returncode): if returncode < 0: signal_num = -returncode @@ -111,6 +159,8 @@ def main(): help="Additional argument to pass through to the Raptor compiler. Repeat as needed.") ap.add_argument("--command-timeout-seconds", type=float, default=1000000.0, help="Per-subprocess timeout in seconds for compiler, runner, and simulation commands.") + ap.add_argument("-j", "--jobs", type=int, default=os.cpu_count() or 1, + help="Number of model validations to run in parallel (default: all available CPUs).") ap.add_argument("--clean", action="store_true", help="Remove generated validation artifacts under each model workspace and exit.") mode_group = ap.add_mutually_exclusive_group() @@ -123,6 +173,8 @@ def main(): ap.add_argument("--verbose", action="store_true", help="Print per-stage progress and subprocess logs for passing validations too.") a = ap.parse_args() + if a.jobs < 1: + ap.error("--jobs must be at least 1") operations_dir = Path(a.operations_dir).resolve() if a.operations_dir else script_dir / "operations" simulator_dir = Path(a.simulator_dir).resolve() if a.simulator_dir else ( @@ -161,6 +213,7 @@ def main(): print(Style.BRIGHT + f"Found {len(onnx_files)} ONNX file(s) to validate." + Style.RESET_ALL) print(f"Operations root: {operations_dir}") + print(f"Parallel jobs: {a.jobs}") print("=" * 72) mode = MODE_FULL @@ -193,43 +246,72 @@ def main(): pass_timing_counts = {label: 0 for _, label in PIM_PASS_LABELS} total_timing_sum = 0.0 timed_benchmark_count = 0 - reporter = ProgressReporter(len(onnx_files), stages_per_model=len(MODE_STAGE_TITLES[mode]), verbose=a.verbose) - for index, onnx_path in enumerate(onnx_files, start=1): - rel = onnx_path.relative_to(operations_dir) - try: - result = validate_network( - onnx_path, a.raptor_path, a.onnx_include_dir, simulator_dir, - crossbar_size=a.crossbar_size, crossbar_count=a.crossbar_count, core_count=a.core_count, - raptor_extra_args=a.raptor_extra_arg, - pimsim_nn_build_dir=pimsim_nn_build_dir, - pimsim_config_path=selected_pimsim_config, - command_timeout_seconds=a.command_timeout_seconds, - threshold=a.threshold, - rtol=a.relative_threshold, - seed=a.seed, - reporter=reporter, - model_index=index, - model_total=len(onnx_files), - verbose=a.verbose, - mode=mode, + reporter = ProgressReporter(len(onnx_files), stages_per_model=1, verbose=a.verbose) + validation_kwargs = { + "raptor_path": a.raptor_path, + "onnx_include_dir": a.onnx_include_dir, + "simulator_dir": simulator_dir, + "crossbar_size": a.crossbar_size, + "crossbar_count": a.crossbar_count, + "core_count": a.core_count, + "raptor_extra_args": a.raptor_extra_arg, + "pimsim_nn_build_dir": pimsim_nn_build_dir, + "pimsim_config_path": selected_pimsim_config, + "command_timeout_seconds": a.command_timeout_seconds, + "threshold": a.threshold, + "rtol": a.relative_threshold, + "seed": a.seed, + } + indexed_files = list(enumerate(onnx_files, start=1)) + workspace_groups = [ + list(group) + for _, group in groupby(indexed_files, key=lambda indexed_path: indexed_path[1].parent) + ] + with TemporaryDirectory(prefix="raptor-validation-") as log_dir: + jobs = [ + ( + [ + ( + index, + onnx_path, + onnx_path.relative_to(operations_dir), + Path(log_dir) / f"{index}.log", + ) + for index, onnx_path in workspace_group + ], + { + "model_total": len(onnx_files), + "mode": mode, + "verbose": a.verbose, + "validation_kwargs": validation_kwargs, + }, ) - results[str(rel)] = result - if result.pim_pass_timings: - benchmark_total = 0.0 - for label, duration in result.pim_pass_timings.items(): - pass_timing_sums[label] += duration - pass_timing_counts[label] += 1 - benchmark_total += duration - total_timing_sum += benchmark_total - timed_benchmark_count += 1 - except subprocess.CalledProcessError as exc: - results[str(rel)] = ValidationResult(False, pimsim_status=PIMSIM_NOT_RUN) - print_validation_error(reporter, rel, exc) - except Exception as exc: - results[str(rel)] = ValidationResult(False, pimsim_status=PIMSIM_NOT_RUN) - print_validation_error(reporter, rel, exc) + for workspace_group in workspace_groups + ] + with ProcessPoolExecutor(max_workers=a.jobs) as executor: + futures = [executor.submit(run_validation_job, job) for job in jobs] + for future in as_completed(futures): + completed_group = future.result() + for rel, result, log_path in completed_group: + reporter.suspend() + output = Path(log_path).read_text(encoding="utf-8", errors="replace") + if output: + print(output, end="" if output.endswith("\n") else "\n") + reporter.resume() + reporter.advance() + reporter.record_result(result.passed) + results[rel] = result + if result.pim_pass_timings: + benchmark_total = 0.0 + for label, duration in result.pim_pass_timings.items(): + pass_timing_sums[label] += duration + pass_timing_counts[label] += 1 + benchmark_total += duration + total_timing_sum += benchmark_total + timed_benchmark_count += 1 reporter.finish() + results = dict(sorted(results.items())) # Summary n_passed = sum(1 for result in results.values() if result.passed)