nnicolosi/Raptor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add ablation study
Validate Operations / validate-operations (push) Has been cancelled
Details

Browse Source 
normalize names and artifact paths
This commit is contained in:
NiccoloN
2026-08-20 17:58:02 +02:00
parent add20e56eb
commit b009e1ff08
67 changed files with 1573 additions and 993 deletions
Download Patch File Download Diff File Expand all files Collapse all files
validation/tools/classification_local_image_validation.py
+14 -11
View File
@@ -16,6 +16,7 @@ if str(VALIDATION_DIR) not in sys.path:
sys.path.insert(0, str(VALIDATION_DIR))
from raptor_validation.onnx_utils import _ONNX_TO_NP, onnx_io, write_inputs_binary, write_inputs_to_memory_bin
from raptor_validation.artifacts import artifacts_dir
from raptor_validation.validate_one import (
MODE_COMPILE_ONLY,
build_dump_ranges,
@@ -75,10 +76,11 @@ def ensure_local_artifacts(args, model_path: Path):
def ensure_existing_artifacts(model_dir: Path):
artifact_root = artifacts_dir(model_dir)
required_paths = [
model_dir / "runner" / "build" / "runner",
model_dir / "raptor" / "pim" / "config.json",
model_dir / "raptor" / "pim" / "memory.bin",
artifact_root / "runner" / "build" / "runner",
artifact_root / "raptor" / "pim" / "config.json",
artifact_root / "raptor" / "pim" / "memory.bin",
]
missing = [str(path) for path in required_paths if not path.exists()]
if missing:
@@ -185,13 +187,13 @@ def draw_classification_panel(image: Image.Image, results, output_path: Path):
def run_reference_and_simulator(args, model_path: Path, tensor: np.ndarray):
model_dir = model_path.parent
runner_build_dir = model_dir / "runner" / "build"
artifact_root = artifacts_dir(model_path.parent)
runner_build_dir = artifact_root / "runner" / "build"
runner_path = runner_build_dir / "runner"
pim_dir = model_dir / "raptor" / "pim"
simulation_dir = model_dir / "classification_demo" / "simulation"
reference_dir = model_dir / "classification_demo" / "reference"
inputs_dir = model_dir / "classification_demo" / "inputs"
pim_dir = artifact_root / "raptor" / "pim"
simulation_dir = artifact_root / "classification_demo" / "simulation"
reference_dir = artifact_root / "classification_demo" / "reference"
inputs_dir = artifact_root / "classification_demo" / "inputs"
simulation_dir.mkdir(parents=True, exist_ok=True)
reference_dir.mkdir(parents=True, exist_ok=True)
@@ -222,7 +224,7 @@ def run_reference_and_simulator(args, model_path: Path, tensor: np.ndarray):
subprocess.run(runner_cmd, cwd=runner_build_dir, check=True)
write_inputs_to_memory_bin(pim_dir / "memory.bin", pim_dir / "config.json", [tensor])
input_bin_path = simulation_dir / "input.bin"
input_bin_path = simulation_dir / "input_0.bin"
write_inputs_binary(input_bin_path, [tensor])
dump_ranges = build_dump_ranges(pim_dir / "config.json", output_descriptors)
output_bin_path = simulation_dir / "out.bin"
@@ -232,7 +234,8 @@ def run_reference_and_simulator(args, model_path: Path, tensor: np.ndarray):
output_bin_path,
dump_ranges,
timeout_sec=args.command_timeout_seconds,
input_paths=[input_bin_path],
input_dir=input_bin_path.parent,
batch_size=1,
)
output_index, output_name, output_dtype_code, output_shape = output_descriptors[0]
validation/tools/conv_lowering_tiles_drawio.py
+22 -22
View File
@@ -37,55 +37,55 @@ class Tile:
TILES = (
Tile("classic-reference", "REFERENCE", "Classic Conv + exact weight unfolding",
Tile("classic-reference", "Reference", "Classic Conv + exact weight unfolding",
"Original OIHW weights; the two implementations choose different K orders.",
"reference", "Y[p,o] = Σc,kh,kw Xpatch[p,c,kh,kw] · W[o,c,kh,kw]",
"At every output position, multiply the patch by one filter and add every product."),
Tile("pimcomp-element", "PIMCOMP", "Element pipeline",
Tile("pimcomp-element", "Pimcomp", "Element pipeline",
"One patch vector per input cycle; mapped weights stay fixed.",
"pimcomp_element", "patchPIM[1×K] · WflatPIM[K×O] → Yp[1×O]",
"Keep Wflat in the arrays; stream one patch each cycle to produce all O outputs."),
Tile("pimcomp-batch", "PIMCOMP", "Batch / replicated pipeline",
Tile("pimcomp-batch", "Pimcomp", "Batch / replicated pipeline",
"Complete Wflat copies divide patches or input samples.",
"pimcomp_batch", "for replica r: Yr = patchr[1×K] · WflatPIM[K×O]",
"Copy all weights R times and send different patches to the copies in parallel."),
Tile("raptor-legacy-im2col", "RAPTOR", "Legacy explicit im2col",
Tile("raptor-legacy-im2col", "Raptor", "Legacy explicit im2col",
"Every patch becomes one row of a global P×K matrix.",
"legacy", "Y[P×O] = im2col(X)[P×K] · WflatR[K×O]",
"Write every image patch as one matrix row, then multiply the two large matrices."),
Tile("raptor-packed-im2col", "RAPTOR", "Packed im2col",
Tile("raptor-packed-im2col", "Raptor", "Packed im2col",
"Pack q patch rows and repeat Wflat on a block diagonal.",
"packed", "packedY[1×qO] = [patch0|…|patchq−1] · diag(WflatR,…,WflatR)",
"Join q patches and use diagonal weight copies so one multiply computes q independent outputs."),
Tile("raptor-streamed-patch", "RAPTOR", "Streamed patch",
Tile("raptor-streamed-patch", "Raptor", "Streamed patch",
"Gather one patch into bounded scratch; avoid global im2col.",
"streamed_patch", "Yp[1×O] = scratchPatchp[1×K] · WflatR[K×O]",
"Gather one patch, multiply it, write its output, and reuse scratch for the next patch."),
Tile("raptor-streamed-packed", "RAPTOR", "Streamed packed",
Tile("raptor-streamed-packed", "Raptor", "Streamed packed",
"Gather q patch rows in bounded scratch, then block-diagonal pack them.",
"streamed_packed", "packedY = packedScratch[1×qK] · diag(WflatR×q)[qK×qO]",
"Gather q patches in small scratch, join them, multiply by diagonal weights, then unpack q outputs."),
Tile("raptor-depthwise", "RAPTOR", "Depthwise special case",
Tile("raptor-depthwise", "Raptor", "Depthwise special case",
"Each channel owns one row-major 3×3 kernel; channels never reduce together.",
"depthwise", "Y[p,c] = Σkh,kw Xpatch[p,c,kh,kw] · W[c,kh,kw]",
"For each channel separately, multiply its nine patch values by its nine weights and add."),
Tile("raptor-output-channel-tiled", "RAPTOR", "Output-channel tiled",
Tile("raptor-output-channel-tiled", "Raptor", "Output-channel tiled",
"Every O tile retains all channel-major K rows and selects output columns.",
"c_tiled", "Y[:,Oj] = patch[1×K] · Wflat[:,Oj][K×|Oj|]; concat j",
"Reuse the full patch for each output-filter group, then join the output groups."),
Tile("raptor-input-k-tiled", "RAPTOR", "Input-K tiled",
Tile("raptor-input-k-tiled", "Raptor", "Input-K tiled",
"Split matching K ranges; add their partial output vectors.",
"k_tiled", "Y[1×O] = Σi patch[Ki] · Wflat[Ki,:]",
"Multiply matching K slices independently, then add their partial output vectors."),
Tile("raptor-tiled-2d", "RAPTOR", "Two-dimensional tiled",
Tile("raptor-tiled-2d", "Raptor", "Two-dimensional tiled",
"Partition both K rows and output-filter columns.",
"tiled_2d", "Y[:,Oj] = Σi patch[Ki] · Wflat[Ki,Oj]; concat j",
"Split both directions: add results down K and join results across output groups."),
Tile("raptor-row-strip", "RAPTOR", "Pixel-major row-strip",
Tile("raptor-row-strip", "Raptor", "Pixel-major row-strip",
"A lane forms patches across one output row and slices K.",
"row_strip", "for x: Y[r,x,:] = Σi patch[r,x,Ki] · Wflat[Ki,:]",
"Move across one output row; at each x form a patch, multiply its K slices, and add."),
Tile("raptor-row-strip-c-tiled", "RAPTOR", "Row-strip + output tiling",
Tile("raptor-row-strip-c-tiled", "Raptor", "Row-strip + output tiling",
"Each row lane is duplicated across disjoint output-column tiles.",
"row_strip_c", "for x,j: Y[r,x,Oj] = patch[r,x,:] · Wflat[:,Oj]",
"Give each output-filter group a copy of the row lane, then join their output columns."),
@@ -473,7 +473,7 @@ def operation_scene(d: Drawio, parent: str, scene: str) -> None:
def reference_body(d: Drawio, parent: str) -> None:
add_box(d, parent, 24, 252, 712, 158, "", fill="#ffffff", stroke=PIMCOMP)
add_text(d, parent, 40, 260, 680, 22,
"PIMCOMP: spatial-major, row-wise positions; C interleaved", 11,
"Pimcomp: Spatial-major, row-wise positions; C interleaved", 11,
color=PIMCOMP, align="center", bold=True)
order_vector(d, parent, 306, "pimcomp", weight=False, label="Input patch")
order_vector(d, parent, 366, "pimcomp", weight=True, label="Matching W")
@@ -483,7 +483,7 @@ def reference_body(d: Drawio, parent: str) -> None:
add_box(d, parent, 24, 424, 712, 158, "", fill="#ffffff", stroke=RAPTOR)
add_text(d, parent, 40, 432, 680, 22,
"RAPTOR: channel-major; each 3×3 plane is row-major", 11,
"Raptor: channel-major; each 3×3 plane is row-major", 11,
color=RAPTOR, align="center", bold=True)
order_vector(d, parent, 478, "raptor", weight=False, label="Input patch")
order_vector(d, parent, 538, "raptor", weight=True, label="Matching W")
@@ -500,22 +500,22 @@ def reference_body(d: Drawio, parent: str) -> None:
def layout_reference(d: Drawio, parent: str, tile: Tile, accent: str) -> None:
if tile.scene == "depthwise":
layout = "independent row-major Kc=9 per channel"
elif tile.owner == "PIMCOMP":
layout = "PIMCOMP spatial-major K order"
elif tile.owner == "Pimcomp":
layout = "Pimcomp Spatial-major K order"
else:
layout = "RAPTOR channel-major K order"
layout = "Raptor channel-major K order"
add_box(d, parent, 24, 140, 712, 42, "", fill=PALE, stroke=accent)
add_text(d, parent, 38, 146, 684, 30,
f"LAYOUT → see REFERENCE tile: {layout}", 10,
f"Layout → see Reference tile: {layout}", 10,
color=accent, align="center", bold=True)
def algorithm_card(d: Drawio, parent: str, tile: Tile, accent: str) -> None:
add_box(d, parent, 24, 638, 712, 102, "", fill="#ffffff", stroke=accent)
add_text(d, parent, 40, 646, 90, 34, "ALGORITHM", 9,
add_text(d, parent, 40, 646, 90, 34, "Algorithm", 9,
color=accent, bold=True)
add_text(d, parent, 132, 644, 588, 38, tile.algorithm, 10)
add_text(d, parent, 40, 690, 90, 34, "MATH", 9,
add_text(d, parent, 40, 690, 90, 34, "Math", 9,
color=accent, bold=True)
add_text(d, parent, 132, 686, 588, 42, tile.formula, 10,
color=accent, bold=True)
@@ -524,7 +524,7 @@ def algorithm_card(d: Drawio, parent: str, tile: Tile, accent: str) -> None:
def render_tile(d: Drawio, tile: Tile, index: int) -> None:
col, row = index % COLS, index // COLS
parent = d.group(col * (TILE + GAP), row * (TILE + GAP), tile.slug)
accent = {"REFERENCE": REFERENCE, "PIMCOMP": PIMCOMP, "RAPTOR": RAPTOR}[tile.owner]
accent = {"Reference": REFERENCE, "Pimcomp": PIMCOMP, "Raptor": RAPTOR}[tile.owner]
add_box(d, parent, 0, 0, TILE, TILE, "", fill="#fbfcff", stroke=accent,
stroke_width=3)
add_box(d, parent, 24, 20, 106, 28, tile.owner, fill=accent, stroke=accent,
validation/tools/pim/ablation/README.md
+66
View File
@@ -0,0 +1,66 @@
# Raptor compiler ablation
`run_ablation.py` performs the complete synchronization/Spatial-planning
ablation study on the Pimcomp model suite. By default it runs `vgg8`,
`resnet18`, `resnet34`, and `googlenet` across `arch-a` and `arch-b`, latency,
and throughput pipeline 4. `arch-c` and `yolo11n` are run only when selected
explicitly.
Latency is pipeline 1; the wrapper invokes the suite runner separately for
latency and throughput pipeline 4.
```bash
.venv/bin/python validation/tools/pim/ablation/run_ablation.py \
--jobs 4
```
## Variants
| Variant | Raptor options |
|---|---|
| `baseline` | None. |
| `no-synchronization` | `--pim-disable-synchronization` |
| `no-spatial-planning` | `--pim-disable-spatial-planning` |
| `no-synchronization-no-spatial-planning` | Both options. |
Every variant runs Raptor only. Pimcomp is not compiled, validated, or
simulated. Reference inputs and outputs are generated once under the shared
common-artifact root and reused by every variant. Ctrl+C terminates the active
variant and all of its worker jobs.
The percentage baseline is `no-synchronization-no-spatial-planning`: both
ablation features are disabled, so its values are `+0.00%`. Every other
variant reports the signed percentage difference of its Raptor metrics from
that reference for the same model, architecture, mode, and pipeline. Positive
values mean the metric is higher; negative values mean it is lower.
## Options
| Option | Description and default |
|---|---|
| `-h`, `--help` | Show help and exit. |
| `--out-dir PATH` | Suite root. Default: `validation/networks/pimcomp_models`. Model artifacts are stored below `<out-dir>/<model>/artifacts`; disabled variants use `<arch>/<mode>[/pipelineN]/ablation/<variant>`. Variant summaries remain under `<out-dir>/ablation/<variant>/`. |
| `--models MODEL [...]` | Models to run. Default: `vgg8 resnet18 resnet34 googlenet`; include `yolo11n` explicitly when needed. |
| `--variant NAME` | Select a variant from the table above; repeat for multiple variants. Default: all variants. The feature-full baseline and percentage reference are added automatically when needed. |
| `--dry-run` | Print the suite-runner commands that would run without modifying files. Default: off. |
All options of
[`run_pimcomp_models.py`](../pimcomp/compare/README.md), including
`--archs`, `--mode`, `--pipeline`, `--no-fast`, and
`--raptor-extra-arg=ARG`, are forwarded to each variant. `--out-dir`,
`--variant`, and `--dry-run` belong to this wrapper; `--only` is reserved for
the suite runner's comparison mode and is rejected by this wrapper, which
always invokes `--raptor-only`. If neither `--mode` nor `--pipeline` is
forwarded, the wrapper uses its default latency and throughput/pipeline-4
case set. Supplying either option overrides that default and is passed through
as one suite-runner invocation per variant.
The feature-full `baseline` variant uses the normal `run_pimcomp_models.py`
artifact paths and is rerun with the same selected cases and forwarded options
as the disabled variants. The three disabled variants are stored below each model's
`artifacts/<arch>/<mode>[/pipelineN]/ablation/` directory. Shared reference
artifacts remain under each model's `artifacts/common` directory. Transient
per-variant comparison summaries are removed after aggregation. The combined
table is written to `<out-dir>/results_ablation.csv`; it contains only the variant, case
identifiers, and signed `latency_percent`, `throughput_percent`,
`power_percent`, and `energy_percent` columns. These are Raptor metrics;
Pimcomp metrics are omitted because the ablation invokes Raptor only.
validation/tools/pim/ablation/run_ablation.py
Executable
+317
View File
@@ -0,0 +1,317 @@
#!/usr/bin/env python3
"""Run the synchronization and Spatial-planning ablation matrix on Pimcomp models."""
from __future__ import annotations
import argparse
import csv
import math
import os
import shlex
import signal
import subprocess
import sys
from dataclasses import dataclass
from pathlib import Path
REPO = Path(__file__).resolve().parents[4]
RUNNER = REPO / "validation/tools/pim/pimcomp/compare/run_pimcomp_models.py"
DEFAULT_OUT_DIR = REPO / "validation/networks/pimcomp_models"
sys.path.insert(0, str(REPO / "validation"))
from raptor_validation.pimcomp_models import (
ABLATION_DEFAULT_MODELS,
add_models_argument,
)
from raptor_validation.artifacts import remove_lock_files
@dataclass(frozen=True)
class Variant:
name: str
raptor_args: tuple[str, ...]
VARIANTS = (
Variant("baseline", ()),
Variant("no-synchronization", ("--pim-disable-synchronization",)),
Variant("no-spatial-planning", ("--pim-disable-spatial-planning",)),
Variant(
"no-synchronization-no-spatial-planning",
("--pim-disable-synchronization", "--pim-disable-spatial-planning"),
),
)
VARIANT_BY_NAME = {variant.name: variant for variant in VARIANTS}
REFERENCE_VARIANT = "no-synchronization-no-spatial-planning"
COMPARISON_RESULTS_FILENAME = "results_comparison.csv"
ABLATION_RESULTS_FILENAME = "results_ablation.csv"
CASE_FIELDS = ("arch", "model", "mode", "raptor_pipeline")
PERCENTAGE_FIELDS = (
("raptor_latency_ms", "latency_percent"),
("raptor_throughput_samples_s", "throughput_percent"),
("raptor_power_mw", "power_percent"),
("raptor_energy_pj", "energy_percent"),
)
RESULT_FIELDS = (*CASE_FIELDS, *(target for _, target in PERCENTAGE_FIELDS))
DEFAULT_CASE_ARGUMENTS = (
("latency", ("--mode", "latency")),
("throughput/pipeline4", ("--mode", "throughput", "--pipeline", "4")),
)
DEFAULT_CASE_KEYS = frozenset({("latency", "1"), ("throughput", "4")})
def parse_args(argv: list[str] | None = None) -> tuple[argparse.Namespace, list[str]]:
parser = argparse.ArgumentParser(
description="Run the complete compiler ablation matrix on the Pimcomp model suite.",
allow_abbrev=False,
)
parser.add_argument(
"--out-dir",
type=Path,
default=DEFAULT_OUT_DIR,
help="Artifact root (default: validation/networks/pimcomp_models).",
)
add_models_argument(parser, ABLATION_DEFAULT_MODELS)
parser.add_argument(
"--variant",
choices=tuple(VARIANT_BY_NAME),
action="append",
dest="variants",
help="Run only this variant; baseline is added when needed. Repeat as needed.",
)
parser.add_argument("--dry-run", action="store_true", help="Print runner commands without modifying files.")
args, forwarded = parser.parse_known_args(argv)
if any(option == "--only" or option.startswith("--only=") for option in forwarded):
parser.error("--only is not supported; the ablation wrapper always runs Raptor only")
return args, forwarded
def selected_variants(names: list[str] | None) -> list[Variant]:
requested = set(names or VARIANT_BY_NAME)
requested.add("baseline")
requested.add(REFERENCE_VARIANT)
return [variant for variant in VARIANTS if variant.name in requested]
def has_option(arguments: list[str], option: str) -> bool:
return any(argument == option or argument.startswith(f"{option}=") for argument in arguments)
def runner_argument_sets(forwarded: list[str]) -> tuple[tuple[str, list[str]], ...]:
if has_option(forwarded, "--mode") or has_option(forwarded, "--pipeline"):
return (("requested", forwarded),)
return tuple(
(label, [*forwarded, *case_arguments])
for label, case_arguments in DEFAULT_CASE_ARGUMENTS
)
def variant_output_dir(out_dir: Path, variant: Variant) -> Path:
return out_dir if variant.name == "baseline" else out_dir / "ablation" / variant.name
def remove_variant_summaries(out_dir: Path) -> None:
summary_root = out_dir / "ablation"
if not summary_root.is_dir():
return
for summary in summary_root.glob("*/results_comparison.csv"):
summary.unlink()
for variant_dir in summary_root.iterdir():
if variant_dir.is_dir() and not any(variant_dir.iterdir()):
variant_dir.rmdir()
if not any(summary_root.iterdir()):
summary_root.rmdir()
def runner_command(
out_dir: Path,
variant: Variant,
models: list[str],
forwarded: list[str],
common_root: Path,
dry_run: bool,
) -> list[str]:
command = [
sys.executable,
str(RUNNER),
"--out-dir",
str(out_dir),
"--raptor-only",
"--models",
*models,
]
if variant.name != "baseline":
command.extend(("--ablation-variant", variant.name))
if not has_option(forwarded, "--common-dir"):
command.extend(("--common-dir", str(common_root)))
command.extend(forwarded)
command.extend(f"--raptor-extra-arg={arg}" for arg in variant.raptor_args)
if dry_run:
command.append("--dry-run")
return command
def terminate_process_group(process: subprocess.Popen[bytes]) -> None:
if process.poll() is not None:
return
try:
os.killpg(process.pid, signal.SIGTERM)
except ProcessLookupError:
return
try:
process.wait(timeout=5)
except subprocess.TimeoutExpired:
try:
os.killpg(process.pid, signal.SIGKILL)
except ProcessLookupError:
return
process.wait()
def percentage_difference(value: str | None, reference: str | None) -> str:
try:
current = float(value) if value is not None else math.nan
baseline = float(reference) if reference is not None else math.nan
except ValueError:
return "NA"
if not math.isfinite(current) or not math.isfinite(baseline) or baseline == 0:
return "NA"
return f"{(current - baseline) / baseline * 100:+.2f}%"
def aggregate_results(
out_dir: Path,
variants: list[Variant],
selected_cases: frozenset[tuple[str, str]] | None = None,
selected_models: frozenset[str] | None = None,
) -> tuple[Path | None, list[str]]:
fields: list[str] | None = None
rows_by_variant: dict[str, list[dict[str, str]]] = {}
failures = []
for variant in variants:
results_path = variant_output_dir(out_dir, variant) / COMPARISON_RESULTS_FILENAME
if not results_path.is_file():
failures.append(f"{variant.name}: missing {results_path}")
continue
with results_path.open(newline="", encoding="utf-8") as stream:
reader = csv.DictReader(stream)
if reader.fieldnames is None:
failures.append(f"{variant.name}: empty {results_path}")
continue
if fields is None:
fields = reader.fieldnames
elif reader.fieldnames != fields:
failures.append(f"{variant.name}: inconsistent columns in {results_path}")
continue
selected_rows = rows_by_variant.setdefault(variant.name, [])
for row in reader:
if selected_models is not None and row.get("model") not in selected_models:
continue
if selected_cases is not None and (
row.get("mode"), row.get("raptor_pipeline")
) not in selected_cases:
continue
selected_rows.append(row)
if fields is None:
return None, failures
reference_rows = {
tuple(row.get(field, "") for field in CASE_FIELDS): row
for row in rows_by_variant.get(REFERENCE_VARIANT, [])
}
if REFERENCE_VARIANT not in rows_by_variant:
failures.append(f"{REFERENCE_VARIANT}: missing percentage reference results")
output = out_dir / ABLATION_RESULTS_FILENAME
with output.open("w", newline="", encoding="utf-8") as stream:
writer = csv.DictWriter(stream, fieldnames=("variant", *RESULT_FIELDS), lineterminator="\n")
writer.writeheader()
for variant in variants:
for row in rows_by_variant.get(variant.name, []):
reference = reference_rows.get(tuple(row.get(field, "") for field in CASE_FIELDS))
writer.writerow(
{
"variant": variant.name,
**{field: row.get(field, "") for field in CASE_FIELDS},
**{
target: percentage_difference(
row.get(source), reference.get(source) if reference else None
)
for source, target in PERCENTAGE_FIELDS
},
}
)
return output, failures
def main(argv: list[str] | None = None) -> int:
args, forwarded = parse_args(argv)
if not RUNNER.is_file():
print(f"Missing Pimcomp runner: {RUNNER}", file=sys.stderr)
return 1
variants = selected_variants(args.variants)
out_dir = args.out_dir.resolve()
case_sets = runner_argument_sets(forwarded)
if args.dry_run:
for variant in variants:
for _, case_forwarded in case_sets:
print(
shlex.join(
runner_command(
out_dir,
variant,
args.models,
case_forwarded,
out_dir,
True,
)
)
)
return 0
out_dir.mkdir(parents=True, exist_ok=True)
failed = []
current_process = None
try:
for variant in variants:
for case_label, case_forwarded in case_sets:
command = runner_command(
out_dir, variant, args.models, case_forwarded, out_dir, False
)
print(f"[{variant.name}/{case_label}] {shlex.join(command)}")
current_process = subprocess.Popen(
command,
cwd=REPO,
start_new_session=True,
)
returncode = current_process.wait()
current_process = None
if returncode:
failed.append(f"{variant.name}/{case_label}: runner exited with {returncode}")
except KeyboardInterrupt:
if current_process is not None:
terminate_process_group(current_process)
remove_lock_files(out_dir)
remove_variant_summaries(out_dir)
print("Interrupted; terminated the active ablation job.", file=sys.stderr)
return 130
remove_lock_files(out_dir)
selected_cases = DEFAULT_CASE_KEYS if len(case_sets) > 1 else None
output, aggregation_failures = aggregate_results(
out_dir,
variants,
selected_cases,
frozenset(args.models),
)
remove_variant_summaries(out_dir)
failed.extend(aggregation_failures)
if output is not None:
print(f"Ablation results: {output}")
if failed:
print("Failed: " + "; ".join(failed), file=sys.stderr)
return 1
return 0
if __name__ == "__main__":
raise SystemExit(main())
validation/tools/pim/pimcomp/compare/README.md
+62
View File
@@ -0,0 +1,62 @@
# Pimcomp model comparison
These scripts compare Raptor with Pimcomp on the models in
`validation/networks/pimcomp_models`.
`run_pimcomp_models.py` runs the supported model suite across the configured
architectures and simulation modes. `compare_raptor_pimcomp_model.py` is the
lower-level one-model comparison used by the suite runner.
## Suite runner
Run all five models, Arch-A and Arch-B, latency, and throughput:
```bash
.venv/bin/python validation/tools/pim/pimcomp/compare/run_pimcomp_models.py
```
### Options
| Option | Description and default |
|---|---|
| `-h`, `--help` | Show help and exit. |
| `--out-dir PATH` | Suite root; artifacts are placed below `<out-dir>/<model>/artifacts/`. Default: `validation/networks/pimcomp_models`. |
| `--common-dir PATH` | Shared root for per-model reference artifacts. Default: `<out-dir>/<model>/artifacts/common`. |
| `--models MODEL [...]` | Models to run: `vgg8`, `resnet18`, `resnet34`, `googlenet`, `yolo11n`. Default: all five; pass a subset to select specific models. |
| `--archs ARCH [...]` | Pimcomp hardware profiles to run. Default: `arch-a arch-b`. |
| `--mode {latency,throughput} [...]` | Simulation modes. Default: both. |
| `--only {raptor,pimcomp}` | Re-run only one compiler side and reuse the other side's existing report. Default: run both sides. |
| `--raptor-only` | Run Raptor without compiling, validating, or simulating Pimcomp. Default: off. |
| `--pipeline {1,2,4,8}` | Select one Raptor pipeline. Default: latency pipeline 1 and throughput pipelines 2, 4, and 8. |
| `--pimsim-time-ms INT` | Throughput convergence deadline. Default: `1000`. |
| `--batch-size INT` | Functional throughput batch size. Default: `128`. |
| `--timeout-seconds FLOAT` | Per-stage timeout; `0` means unlimited. Default: `0`. |
| `-j INT`, `--jobs INT` | Parallel comparison workers. Default: `4`. |
| `--clean` | Remove generated comparison artifacts and summaries, then exit. Default: off. |
| `--ablation-variant NAME` | Put generated artifacts below `<mode>[/pipelineN]/ablation/NAME` and write the summary below `ablation/NAME`. Default: none. |
| `--dry-run` | Print commands without modifying files. Default: off. |
| `--no-fast` | Disable fast throughput convergence. Default: off. |
| `--raptor-extra-arg=ARG` | Extra Raptor compiler argument; repeat for multiple arguments. Default: none. |
Arguments beginning with `--` must use the equals form when passed through:
```bash
.venv/bin/python validation/tools/pim/pimcomp/compare/run_pimcomp_models.py \
--models vgg8 \
--raptor-extra-arg=--pim-disable-synchronization
```
The runner writes `results_comparison.csv` under the selected result root. It reuses
shared model inputs, outputs, and reference runners, and reuses Pimcomp
artifacts between pipelines in the same model/architecture/mode group. With
`--raptor-only`, only Raptor results are generated and Pimcomp is not invoked.
All generated files are kept below each model's `artifacts/` directory;
`--clean` also removes any `.lock` files left by an interrupted reference
generation.
## One-model comparator
`compare_raptor_pimcomp_model.py` compares one ONNX model. Its required
arguments are `--model PATH` and `--out-dir PATH`; use `--help` for the full
lower-level interface. The suite runner supplies the model, hardware profile,
simulation mode, and reuse paths automatically.
validation/tools/pim/pimcomp/compare/compare_raptor_pimcomp.py → validation/tools/pim/pimcomp/compare/compare_raptor_pimcomp_model.py
+243 -150
View File
@@ -2,6 +2,7 @@
from __future__ import annotations
import argparse
import fcntl
import gzip
import importlib.util
import json
@@ -30,8 +31,8 @@ PIMSIM_CONFIG_DIR = VALIDATION_DIR / "pimsim_configs/pimcomp"
PIMCOMP_OUTPUT_FILES = ("SimulationInfo.gz", "VerificationInfo.json", "MappingResult.txt")
sys.path.insert(0, str(VALIDATION_DIR))
from raptor_validation.gen_network_runner import gen_network_runner # noqa: E402
from raptor_validation.onnx_utils import ( # noqa: E402
from raptor_validation.gen_network_runner import gen_network_runner
from raptor_validation.onnx_utils import (
_ONNX_TO_NP,
generate_input_batch,
gen_random_inputs,
@@ -41,12 +42,13 @@ from raptor_validation.onnx_utils import ( # noqa: E402
write_input_batch_csv,
write_inputs_to_memory_bin,
)
from raptor_validation.raptor import compile_with_raptor # noqa: E402
from raptor_validation.pimsim_nn import ( # noqa: E402
from raptor_validation.raptor import compile_with_raptor
from raptor_validation.pimsim_nn import (
export_raptor_pimsim_artifact,
parse_pimsim_nn_metrics,
)
from raptor_validation.validate_one import ( # noqa: E402
from raptor_validation.artifacts import runner_uses_library
from raptor_validation.validate_one import (
STAGE_COLORS,
build_pim_simulator_command,
build_dump_ranges,
@@ -99,7 +101,7 @@ def print_step(
stage: str | None = None,
):
color = STAGE_COLORS.get(stage or name, Fore.WHITE)
print("\n" + Style.BRIGHT + color + f"[{name}]" + Style.RESET_ALL)
print(Style.BRIGHT + color + f"[{name}]" + Style.RESET_ALL)
if cmd is not None:
print(f" cwd: {cwd or REPO}")
print(f" $ {shell_join(cmd)}")
@@ -134,7 +136,7 @@ def exception_message(exc: BaseException) -> str:
def print_failure(name: str, exc: BaseException | str) -> None:
message = exc if isinstance(exc, str) else exception_message(exc)
print(
"\n" + Style.BRIGHT + Fore.RED + f"[{name} FAILED]" + Style.RESET_ALL,
Style.BRIGHT + Fore.RED + f"[{name} FAILED]" + Style.RESET_ALL,
file=sys.stderr,
)
for line in message.splitlines()[:20]:
@@ -239,6 +241,17 @@ def reference_inputs_exist(
)
def reference_batch_dirs(root: Path, batch_size: int) -> list[Path]:
return [root / f"batch_{index:06d}" for index in range(batch_size)]
def reference_batch_outputs_exist(
outputs_desc: list[tuple[int, str, int, list[int]]],
reference_dirs: list[Path],
) -> bool:
return all(reference_outputs_exist(outputs_desc, reference_dir) for reference_dir in reference_dirs)
def compare_simulator_outputs(
output_bin: Path,
outputs_desc: list[tuple[int, str, int, list[int]]],
@@ -308,7 +321,7 @@ def select_pimsim_config(args: argparse.Namespace, hardware: dict[str, int]) ->
if args.pimsim_mode == "latency" or config["sim_config"]["sim_time"] == args.pimsim_time_ms:
return path
raise ValueError(
f"No pre-generated {args.pimsim_mode} pimsim-nn config for {args.pimsim_time_ms} ms matches {hardware}"
f"No pre-generated {args.pimsim_mode} Pimsim config for {args.pimsim_time_ms} ms matches {hardware}"
)
@@ -329,13 +342,14 @@ def compile_reference(
runner_dir = work_dir / "runner"
build_dir = runner_dir / "build"
raptor_dir.mkdir(parents=True, exist_ok=True)
shutil.rmtree(build_dir, ignore_errors=True)
build_dir.mkdir(parents=True, exist_ok=True)
stem = model_path.stem
onnx_ir_base = raptor_dir / stem
runner_base = runner_dir / stem
run_logged(
"Compile Reference ONNX IR",
"Compile reference ONNX IR",
[str(args.raptor_path), str(model_path), "-o", str(onnx_ir_base), "--EmitONNXIR",
"--mlir-elide-elementsattrs-if-larger=16", "--enable-conv-opt-pass=false"],
cwd=REPO,
@@ -344,7 +358,7 @@ def compile_reference(
stage="Compile ONNX",
)
run_logged(
"Compile Reference Native",
"Compile reference native",
[str(args.raptor_path), "-O3", str(model_path), "-o", str(runner_base)],
cwd=REPO,
timeout_sec=args.timeout_seconds,
@@ -353,7 +367,7 @@ def compile_reference(
)
network_so = runner_base.with_suffix(".so")
print_step("Generate Runner Source", stage="Build Runner")
print_step("Generate runner source", stage="Build runner")
gen_network_runner(
model_path,
network_so,
@@ -364,15 +378,15 @@ def compile_reference(
)
run_logged(
"Configure Runner",
"Configure runner",
["cmake", str(runner_dir), "-DCMAKE_BUILD_TYPE=Release", "-DCMAKE_C_FLAGS_RELEASE=-O3"],
cwd=build_dir,
timeout_sec=args.timeout_seconds,
steps=steps,
stage="Build Runner",
stage="Build runner",
)
run_logged(
"Build Runner",
"Build runner",
["cmake", "--build", ".", "-j"],
cwd=build_dir,
timeout_sec=args.timeout_seconds,
@@ -382,6 +396,20 @@ def compile_reference(
return build_dir / "runner"
def ensure_reference_runner(
args: argparse.Namespace,
model_path: Path,
work_dir: Path,
steps: list[StepRecord],
) -> Path:
runner_dir = work_dir / "runner"
runner_path = runner_dir / "build/runner"
library_path = runner_dir / f"{model_path.stem}.so"
if runner_uses_library(runner_path, library_path):
return runner_path
return compile_reference(args, model_path, work_dir, steps)
def generate_reference_outputs(
runner_path: Path,
runner_build_dir: Path,
@@ -390,6 +418,8 @@ def generate_reference_outputs(
steps: list[StepRecord],
args: argparse.Namespace,
out_dir: Path,
*,
print_header: bool = True,
) -> Path:
inputs_dir = out_dir / "inputs"
reference_dir = out_dir / "outputs"
@@ -397,11 +427,12 @@ def generate_reference_outputs(
reference_dir.mkdir(parents=True, exist_ok=True)
flags, _ = save_inputs_to_files(model_path, arrays_in_order, inputs_dir)
run_logged(
"Run Reference",
"Run reference",
[str(runner_path), *flags, "--save-csv-dir", str(reference_dir)],
cwd=runner_build_dir,
timeout_sec=args.timeout_seconds,
steps=steps,
print_header=print_header,
)
return reference_dir
@@ -414,19 +445,56 @@ def generate_reference_batch_outputs(
steps: list[StepRecord],
args: argparse.Namespace,
out_dir: Path,
*,
print_header: bool = True,
) -> list[Path]:
return [
generate_reference_outputs(
if print_header:
print_step("Run reference")
references = []
for index, sample in enumerate(input_batch):
references.append(
generate_reference_outputs(
runner_path,
runner_build_dir,
model_path,
sample,
steps,
args,
out_dir / f"batch_{index:06d}",
print_header=False,
)
)
return references
def prepare_reference_batch_outputs(
runner_path: Path,
runner_build_dir: Path,
model_path: Path,
input_batch: list[list[np.ndarray]],
outputs_desc: list[tuple[int, str, int, list[int]]],
steps: list[StepRecord],
args: argparse.Namespace,
out_dir: Path,
*,
print_header: bool = True,
) -> list[Path]:
reference_dirs = reference_batch_dirs(out_dir, len(input_batch))
lock_path = out_dir.parent / ".reference.lock"
with lock_path.open("w", encoding="utf-8") as lock:
fcntl.flock(lock.fileno(), fcntl.LOCK_EX)
if reference_batch_outputs_exist(outputs_desc, reference_dirs):
return reference_dirs
return generate_reference_batch_outputs(
runner_path,
runner_build_dir,
model_path,
sample,
input_batch,
steps,
args,
out_dir / f"batch_{index:06d}",
out_dir,
print_header=print_header,
)
for index, sample in enumerate(input_batch)
]
def prepare_common_artifacts(
@@ -437,10 +505,8 @@ def prepare_common_artifacts(
inputs_desc, outputs_desc, arrays_in_order = load_model_inputs(model_path, args.seed)
inputs_dir = common_dir / "inputs"
outputs_dir = common_dir / "outputs"
runner_path = common_dir / "runner/build/runner"
steps: list[StepRecord] = []
if not runner_path.exists():
runner_path = compile_reference(args, model_path, common_dir, steps)
runner_path = ensure_reference_runner(args, model_path, common_dir, steps)
inputs_ready = reference_inputs_exist(inputs_desc, inputs_dir)
outputs_ready = reference_outputs_exist(outputs_desc, outputs_dir)
@@ -478,7 +544,7 @@ def compile_raptor_target(
"--pim-emit-json",
*args.raptor_extra_arg,
]
print_step("Compile Raptor", cmd, REPO, "Compile PIM")
print_step("Compile Raptor", cmd, REPO, "Compile Pim")
start = time.perf_counter()
command = shell_join(cmd)
raptor_extra_args = [
@@ -503,7 +569,7 @@ def compile_raptor_target(
except Exception as exc:
steps.append(
StepRecord(
name="Compile Raptor PIM",
name="Compile Raptor Pim",
duration_sec=time.perf_counter() - start,
command=command,
status="failed",
@@ -514,7 +580,7 @@ def compile_raptor_target(
steps.append(
StepRecord(
name="Compile Raptor PIM",
name="Compile Raptor Pim",
duration_sec=time.perf_counter() - start,
command=command,
)
@@ -548,7 +614,8 @@ def run_functional_validation(
pim_dir,
output_bin,
dump_ranges,
input_bins,
input_bins[0].parent,
batch_size,
args.pimsim_mode,
batch_output_dir,
)
@@ -559,7 +626,7 @@ def run_functional_validation(
cwd=args.pim_simulator_dir,
timeout_sec=args.timeout_seconds,
steps=steps,
stage="Run Functional Simulation",
stage="Run functional simulation",
)
max_diffs: dict[str, float] = {}
failed_iterations = []
@@ -610,9 +677,9 @@ def compile_pimcomp(
(pimcomp_output_dir / name).unlink(missing_ok=True)
model_name = args.pimcomp_model_name or f"compare_{model_path.stem}"
frontend_json = frontend_json_dir / f"{model_name}.json"
print_step("Compile PIMCOMP", stage="Compile PIM")
# The original PIMCOMP frontend rewrites its input ONNX while loading it.
# Isolate that mutation without changing the model sent through PIMCOMP.
print_step("Compile Pimcomp", stage="Compile Pim")
# The original Pimcomp frontend rewrites its input ONNX while loading it.
# Isolate that mutation without changing the model sent through Pimcomp.
with TemporaryDirectory(prefix="pimcomp-model-") as temp_dir:
frontend_model = Path(temp_dir) / model_path.name
shutil.copy2(model_path, frontend_model)
@@ -625,12 +692,12 @@ def compile_pimcomp(
str(frontend_json),
]
run_logged(
"Compile PIMCOMP Frontend",
"Compile Pimcomp frontend",
frontend_cmd,
cwd=args.pimcomp_dir / "frontend",
timeout_sec=args.timeout_seconds,
steps=steps,
stage="Compile PIM",
stage="Compile Pim",
print_header=False,
)
backend_cmd = [
@@ -643,12 +710,12 @@ def compile_pimcomp(
]
try:
run_logged(
"Compile PIMCOMP Backend",
"Compile Pimcomp backend",
backend_cmd,
cwd=frontend_json_dir.parent,
timeout_sec=args.timeout_seconds,
steps=steps,
stage="Compile PIM",
stage="Compile Pim",
print_header=False,
)
finally:
@@ -666,7 +733,7 @@ def export_pimcomp_for_pimsim_nn(simulation_info: Path, output_dir: Path) -> Pat
sim_config = sim_info["config"]
core_count = int(sim_config["core_cnt"])
if core_count <= 0:
raise ValueError("PIMCOMP SimulationInfo.gz must configure at least one core")
raise ValueError("Pimcomp SimulationInfo.gz must configure at least one core")
core_indices = range(core_count)
config = {
@@ -709,7 +776,7 @@ def export_pimcomp_for_rust(
output_dir: Path,
) -> Path:
if len(runtime_inputs) != 1:
raise ValueError("PIMCOMP export currently requires exactly one runtime input tensor")
raise ValueError("Pimcomp export currently requires exactly one runtime input tensor")
if output_dir.exists():
shutil.rmtree(output_dir)
exporter = load_pimcomp_exporter()
@@ -901,7 +968,7 @@ def export_pimcomp_for_rust(
instructions.append(translated)
out_offset += width
else:
raise RuntimeError(f"Unsupported PIMCOMP op {op}")
raise RuntimeError(f"Unsupported Pimcomp op {op}")
with open(output_dir / f"core_{core_idx}.json", "w", encoding="utf-8") as f:
json.dump(instructions, f, separators=(",", ":"))
@@ -935,7 +1002,7 @@ def run_pimsim_nn(
cwd=args.pimsim_nn_build_dir,
timeout_sec=args.timeout_seconds * 10.0,
steps=steps,
stage="Run Non-functional Simulation",
stage="Run non-functional simulation",
)
return parse_pimsim_nn_metrics(output)
@@ -1091,12 +1158,9 @@ def restore_side_records(
def default_common_dir(out_dir: Path) -> Path:
if out_dir.name == "latency":
return out_dir.parents[1] / "common"
if out_dir.parent.name == "throughput":
return out_dir.parents[2] / "common"
if out_dir.name.startswith("arch-"):
return out_dir.parent / "common"
for parent in (out_dir, *out_dir.parents):
if parent.name == "artifacts":
return parent / "common"
return out_dir / "common"
@@ -1155,19 +1219,19 @@ def write_report(
):
report_path.parent.mkdir(parents=True, exist_ok=True)
lines = [
"# Raptor vs PIMCOMP Comparison Report",
"# Raptor vs Pimcomp comparison report",
"",
f"- Model: `{model_path}`",
f"- Hardware: `{hardware.get('core_count', 'n/a')} cores`, `{hardware.get('crossbar_count', 'n/a')} xbars/core`, `{hardware.get('crossbar_size', 'n/a')}x{hardware.get('crossbar_size', 'n/a')}` crossbars, mesh `{hardware.get('mesh_rows', 'n/a')}x{hardware.get('mesh_cols', 'n/a')}`",
f"- PIMCOMP pipeline: `{pimcomp_pipeline}`",
f"- PIMCOMP replication: `{pimcomp_replication}`",
f"- Pimcomp pipeline: `{pimcomp_pipeline}`",
f"- Pimcomp replication: `{pimcomp_replication}`",
"",
]
if failures or any(step.status != "passed" for step in steps):
lines.extend(
[
"## Failures / Skipped Work",
"## Failures / skipped work",
"",
"The script did not abort. The failed stage was recorded and any dependent stage was skipped when its inputs were not available.",
"",
@@ -1182,21 +1246,21 @@ def write_report(
lines.extend(
[
"## Functional Validation",
"## Functional validation",
"",
f"- Raptor via `pim-simulator`: `{validation_status(raptor_validation)}`",
f"- PIMCOMP via exported `pim-simulator`: `{validation_status(pimcomp_validation)}`",
f"- Pimcomp via exported `pim-simulator`: `{validation_status(pimcomp_validation)}`",
]
)
if raptor_validation.error:
lines.append(f"- Raptor validation note: `{raptor_validation.error.splitlines()[0]}`")
if pimcomp_validation.error:
lines.append(f"- PIMCOMP validation note: `{pimcomp_validation.error.splitlines()[0]}`")
lines.append(f"- Pimcomp validation note: `{pimcomp_validation.error.splitlines()[0]}`")
lines.extend(["", "### Max Output Differences", ""])
lines.extend(["", "### Max output differences", ""])
diff_names = sorted(set(raptor_validation.max_diffs) | set(pimcomp_validation.max_diffs))
if diff_names:
lines.extend(["| Output | Raptor max diff | PIMCOMP max diff |", "|---|---:|---:|"])
lines.extend(["| Output | Raptor max diff | Pimcomp max diff |", "|---|---:|---:|"])
for name in diff_names:
lines.append(
f"| `{name}` | {raptor_validation.max_diffs.get(name, float('nan')):.6e} | "
@@ -1208,7 +1272,7 @@ def write_report(
lines.extend(
[
"",
"## pimsim-nn Performance",
"## Pimsim performance",
"",
f"- Mode: `{pimsim_mode}`",
"",
@@ -1221,7 +1285,7 @@ def write_report(
"|---|---|---:|---:|---:|---:|---:|",
f"| Raptor | {perf_status(raptor_perf)} | {perf_value(raptor_perf, 'average_latency_ms', 'ms')} | {perf_value(raptor_perf, 'throughput', 'samples/s')} | "
f"{perf_value(raptor_perf, 'average_power_mw', 'mW')} | {perf_value(raptor_perf, 'average_energy_pj', 'pJ/it')} | {perf_value(raptor_perf, 'output_count')} |",
f"| PIMCOMP | {perf_status(pimcomp_perf)} | {perf_value(pimcomp_perf, 'average_latency_ms', 'ms')} | {perf_value(pimcomp_perf, 'throughput', 'samples/s')} | "
f"| Pimcomp | {perf_status(pimcomp_perf)} | {perf_value(pimcomp_perf, 'average_latency_ms', 'ms')} | {perf_value(pimcomp_perf, 'throughput', 'samples/s')} | "
f"{perf_value(pimcomp_perf, 'average_power_mw', 'mW')} | {perf_value(pimcomp_perf, 'average_energy_pj', 'pJ/it')} | {perf_value(pimcomp_perf, 'output_count')} |",
"",
]
@@ -1233,40 +1297,40 @@ def write_report(
"|---|---|---:|---:|---:|",
f"| Raptor | {perf_status(raptor_perf)} | {perf_value(raptor_perf, 'latency_ms', 'ms')} | "
f"{perf_value(raptor_perf, 'average_power_mw', 'mW')} | {perf_value(raptor_perf, 'average_energy_pj', 'pJ')} |",
f"| PIMCOMP | {perf_status(pimcomp_perf)} | {perf_value(pimcomp_perf, 'latency_ms', 'ms')} | "
f"| Pimcomp | {perf_status(pimcomp_perf)} | {perf_value(pimcomp_perf, 'latency_ms', 'ms')} | "
f"{perf_value(pimcomp_perf, 'average_power_mw', 'mW')} | {perf_value(pimcomp_perf, 'average_energy_pj', 'pJ')} |",
"",
]
)
if raptor_perf.get("reason") or raptor_perf.get("error"):
lines.append(f"- Raptor pimsim-nn note: `{(raptor_perf.get('reason') or raptor_perf.get('error')).splitlines()[0]}`")
lines.append(f"- Raptor Pimsim note: `{(raptor_perf.get('reason') or raptor_perf.get('error')).splitlines()[0]}`")
if pimcomp_perf.get("reason") or pimcomp_perf.get("error"):
lines.append(f"- PIMCOMP pimsim-nn note: `{(pimcomp_perf.get('reason') or pimcomp_perf.get('error')).splitlines()[0]}`")
lines.append(f"- Pimcomp Pimsim note: `{(pimcomp_perf.get('reason') or pimcomp_perf.get('error')).splitlines()[0]}`")
if lines[-1] != "":
lines.append("")
lines.extend(
[
"## Instruction Summary",
"## Instruction summary",
"",
"| Compiler | Status | Active cores | Total instructions | Sends | Receives | MVMUL |",
"|---|---|---:|---:|---:|---:|---:|",
f"| Raptor | {'FAILED' if raptor_instr.get('error') else 'SKIPPED' if raptor_instr.get('skipped') else 'DONE'} | {raptor_instr.get('active_cores', 0)} | {raptor_instr.get('total_instructions', 0)} | {raptor_instr.get('op_counts', {}).get('send', 0)} | {raptor_instr.get('op_counts', {}).get('recv', 0)} | {raptor_instr.get('op_counts', {}).get('mvmul', 0)} |",
f"| PIMCOMP | {'FAILED' if pimcomp_instr.get('error') else 'SKIPPED' if pimcomp_instr.get('skipped') else 'DONE'} | {pimcomp_instr.get('active_cores', 0)} | {pimcomp_instr.get('total_instructions', 0)} | {pimcomp_instr.get('op_counts', {}).get('send', 0)} | {pimcomp_instr.get('op_counts', {}).get('recv', 0)} | {pimcomp_instr.get('op_counts', {}).get('mvmul', 0)} |",
f"| Pimcomp | {'FAILED' if pimcomp_instr.get('error') else 'SKIPPED' if pimcomp_instr.get('skipped') else 'DONE'} | {pimcomp_instr.get('active_cores', 0)} | {pimcomp_instr.get('total_instructions', 0)} | {pimcomp_instr.get('op_counts', {}).get('send', 0)} | {pimcomp_instr.get('op_counts', {}).get('recv', 0)} | {pimcomp_instr.get('op_counts', {}).get('mvmul', 0)} |",
"",
"### Raptor Op Distribution",
"### Raptor op distribution",
"",
"| Op | Count | Share |",
"|---|---:|---:|",
*format_op_table(raptor_instr.get("op_counts", {}), raptor_instr.get("total_instructions", 0)),
"",
"### PIMCOMP Op Distribution",
"### Pimcomp op distribution",
"",
"| Op | Count | Share |",
"|---|---:|---:|",
*format_op_table(pimcomp_instr.get("op_counts", {}), pimcomp_instr.get("total_instructions", 0)),
"",
"## Step Timings",
"## Step timings",
"",
"| Step | Status | Duration (s) | Return code |",
"|---|---|---:|---:|",
@@ -1279,7 +1343,7 @@ def write_report(
)
failed_steps = [step for step in steps if step.status != "passed"]
if failed_steps:
lines.extend(["", "### Failed Step Details", ""])
lines.extend(["", "### Failed step details", ""])
for step in failed_steps:
lines.extend(
[
@@ -1294,7 +1358,7 @@ def write_report(
lines.append("")
if raptor_pass_timings:
lines.extend(["", "## Raptor Pass Timings", "", "| Pass | Duration (s) |", "|---|---:|"])
lines.extend(["", "## Raptor pass timings", "", "| Pass | Duration (s) |", "|---|---:|"])
for name, duration in raptor_pass_timings.items():
lines.append(f"| {name} | {duration:.4f} |")
report_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
@@ -1325,7 +1389,7 @@ def main():
parser.add_argument(
"--pimcomp-config",
type=Path,
help="PIMCOMP hardware config (default: <pimcomp-dir>/config.json).",
help="Pimcomp hardware config (default: <pimcomp-dir>/config.json).",
)
parser.add_argument("--pim-simulator-dir", default=REPO / "backend-simulators/pim/pim-simulator", type=Path)
parser.add_argument("--pimsim-nn-build-dir", default=REPO / "backend-simulators/pim/pimsim-nn/build", type=Path)
@@ -1347,7 +1411,7 @@ def main():
parser.add_argument("--pimsim-mode", choices=["latency", "throughput"], default="latency")
parser.add_argument("--batch-size", type=int, default=128)
parser.add_argument("--pimcomp-pipeline", choices=["element", "batch"])
parser.add_argument("--pimcomp-model-name", help="Use a PIMCOMP built-in model name such as vgg16.")
parser.add_argument("--pimcomp-model-name", help="Use a Pimcomp built-in model name such as vgg16.")
parser.add_argument(
"--pimcomp-replication",
choices=["balance", "W0H0", "uniform", "GA"],
@@ -1361,27 +1425,41 @@ def main():
parser.add_argument(
"--reuse-pimcomp-dir",
type=Path,
help="Reuse a directory containing PIMCOMP SimulationInfo.gz, VerificationInfo.json, and MappingResult.txt.",
help="Reuse a directory containing Pimcomp SimulationInfo.gz, VerificationInfo.json, and MappingResult.txt.",
)
parser.add_argument(
"--reuse-pimcomp-report",
type=Path,
help="Preserve the PIMCOMP side of an existing comparison report without rerunning it.",
help="Preserve the Pimcomp side of an existing comparison report without rerunning it.",
)
parser.add_argument("--skip-pimsim-nn", action="store_true")
parser.add_argument(
"--no-fast",
action="store_true",
help="Disable fast pimsim-nn throughput convergence for authoritative experiments.",
help="Disable fast Pimsim throughput convergence for authoritative experiments.",
)
parser.add_argument("--verbose-raptor-compile", action="store_true")
parser.add_argument("--raptor-extra-arg", action="append", default=[])
parser.add_argument(
"--raptor-only",
action="store_true",
help="Run Raptor compilation, validation, and simulation without running Pimcomp.",
)
parser.add_argument(
"--fail-on-error",
action="store_true",
help="Return a non-zero status if a stage or semantic validation fails.",
)
args = parser.parse_args()
if args.raptor_only and any(
option is not None
for option in (
args.reuse_raptor_report,
args.reuse_pimcomp_dir,
args.reuse_pimcomp_report,
)
):
parser.error("--raptor-only cannot be combined with report reuse options")
if args.reuse_pimcomp_dir is not None and args.reuse_pimcomp_report is not None:
parser.error("--reuse-pimcomp-dir and --reuse-pimcomp-report are mutually exclusive")
if args.pimsim_time_ms <= 0:
@@ -1439,6 +1517,7 @@ def main():
runner_path: Path | None = None
reference_dir: Path | None = None
reference_dirs: list[Path] = []
reference_stage_started = False
raptor_pim_dir: Path | None = None
raptor_pimsim_dir: Path | None = None
raptor_pass_timings: dict[str, float] = {}
@@ -1451,11 +1530,19 @@ def main():
reuse_raptor = args.reuse_raptor_report is not None
raptor_validation = skipped_validation("Raptor validation did not run")
pimcomp_validation = failed_validation("PIMCOMP validation did not run")
raptor_perf: dict[str, Any] = skipped_perf("pimsim-nn Raptor did not run")
pimcomp_perf: dict[str, Any] = skipped_perf("pimsim-nn PIMCOMP did not run")
pimcomp_validation = (
skipped_validation("Skipped by --raptor-only")
if args.raptor_only
else failed_validation("Pimcomp validation did not run")
)
raptor_perf: dict[str, Any] = skipped_perf("Pimsim Raptor did not run")
pimcomp_perf: dict[str, Any] = skipped_perf(
"Skipped by --raptor-only" if args.raptor_only else "Pimsim Pimcomp did not run"
)
raptor_instr: dict[str, Any] = empty_instruction_summary("Raptor instruction parsing did not run")
pimcomp_instr: dict[str, Any] = empty_instruction_summary("PIMCOMP instruction parsing did not run")
pimcomp_instr: dict[str, Any] = empty_instruction_summary(
"Skipped by --raptor-only" if args.raptor_only else "Pimcomp instruction parsing did not run"
)
loaded_hardware = try_stage(failures, "Load hardware configuration", load_effective_hardware, args)
if loaded_hardware is not None:
@@ -1476,22 +1563,21 @@ def main():
if args.reuse_pimcomp_dir is not None:
reused_pimcomp_report_path = args.reuse_pimcomp_dir.resolve().parent / "comparison_report.json"
reuse_pimcomp = reused_pimcomp_report_path is not None
run_pimcomp = not args.raptor_only and not reuse_pimcomp
if reuse_pimcomp:
reused_pimcomp_report_path = reused_pimcomp_report_path.resolve()
if not reused_pimcomp_report_path.exists():
raise ValueError(f"Missing PIMCOMP report: {reused_pimcomp_report_path}")
raise ValueError(f"Missing Pimcomp report: {reused_pimcomp_report_path}")
with open(reused_pimcomp_report_path, "r", encoding="utf-8") as f:
reused_pimcomp = json.load(f)
if reused_pimcomp.get("pimcomp_model_source") != "original_onnx":
raise ValueError("Reused PIMCOMP artifacts were not generated from the original ONNX model")
raise ValueError("Reused Pimcomp artifacts were not generated from the original ONNX model")
pimcomp_validation = CompareResult(**reused_pimcomp["pimcomp_validation"])
pimcomp_perf = reused_pimcomp["pimcomp_performance"]
pimcomp_instr = reused_pimcomp["pimcomp_instruction_summary"]
simulation_info = path_from_report(reused_pimcomp, "pimcomp_simulation_info")
pimcomp_export_dir = path_from_report(reused_pimcomp, "pimcomp_exported_pim")
pimcomp_pimsim_dir = path_from_report(reused_pimcomp, "pimcomp_pimsim_nn")
if pimcomp_pimsim_dir is None and (reused_pimcomp_report_path.parent / "pimsim_nn").is_dir():
pimcomp_pimsim_dir = reused_pimcomp_report_path.parent / "pimsim_nn"
pimsim_config = path_from_report(reused_pimcomp, "pimsim_config")
restore_side_records(reused_pimcomp, "pimcomp", failures, steps)
@@ -1504,6 +1590,9 @@ def main():
raise ValueError(f"Reused Raptor hardware differs: {reused_hardware} != {hardware}")
reference_dir = common_dir / "outputs"
runner_path = path_from_report(reused, "reference_runner")
expected_library = common_dir / "runner" / f"{functional_model_path.stem}.so"
if runner_path is None or not runner_uses_library(runner_path, expected_library):
runner_path = ensure_reference_runner(args, functional_model_path, common_dir, steps)
raptor_pim_dir = path_from_report(reused, "raptor_pim")
raptor_pimsim_dir = path_from_report(reused, "raptor_pimsim_nn")
arrays_in_order = load_saved_inputs(
@@ -1519,7 +1608,6 @@ def main():
print_step("Reuse Raptor")
print(f" Report: {reuse_report_path}")
expected_runner_path = common_dir / "runner/build/runner"
common_inputs_dir = common_dir / "inputs"
common_reference_dir = common_dir / "outputs"
@@ -1539,20 +1627,15 @@ def main():
else:
inputs_ready = False
if expected_runner_path.exists():
runner_path = expected_runner_path
else:
reference_compile = try_stage(
failures,
"Compile reference",
compile_reference,
args,
functional_model_path,
common_dir,
steps,
)
if reference_compile is not None:
runner_path = reference_compile
runner_path = try_stage(
failures,
"Prepare reference runner",
ensure_reference_runner,
args,
functional_model_path,
common_dir,
steps,
)
if runner_path is not None and runner_path.exists() and model_io is not None:
if inputs_ready and reference_outputs_exist(outputs_desc, common_reference_dir):
@@ -1570,9 +1653,11 @@ def main():
steps,
args,
common_dir,
print_header=not reference_stage_started,
)
if generated_reference is not None:
reference_dir = generated_reference
reference_stage_started = True
else:
record_failure(
failures,
@@ -1589,28 +1674,32 @@ def main():
write_input_batch_csv(out_dir / "inputs.csv", input_batch)
raptor_input_bins = write_input_batch_binaries(input_batch, out_dir / "simulation/raptor_inputs")
if args.pimsim_mode == "throughput":
batch_reference_dir = common_dir / f"reference_batch_{args.batch_size}_seed_{args.seed}"
throughput_references = try_stage(
failures,
"Run throughput references",
generate_reference_batch_outputs,
"Run reference",
prepare_reference_batch_outputs,
runner_path,
runner_path.parent,
functional_model_path,
input_batch,
outputs_desc,
steps,
args,
out_dir / "reference",
batch_reference_dir,
print_header=not reference_stage_started,
) if runner_path is not None and runner_path.exists() else None
if throughput_references is not None:
reference_dirs = throughput_references
reference_dir = out_dir / "reference"
reference_dir = batch_reference_dir
reference_stage_started = True
elif reference_dir is not None:
reference_dirs = [reference_dir]
if not reuse_raptor and model_path.exists() and hardware["core_count"] > 0:
compiled_raptor = try_stage(
failures,
"Compile Raptor PIM",
"Compile Raptor Pim",
compile_raptor_target,
model_path,
out_dir / "raptor",
@@ -1623,8 +1712,8 @@ def main():
elif not reuse_raptor:
record_failure(
failures,
"Skip Raptor PIM compile",
"Raptor PIM compile was skipped because the ONNX model or hardware configuration is not available.",
"Skip Raptor Pim compile",
"Raptor Pim compile was skipped because the ONNX model or hardware configuration is not available.",
)
raptor_functional_pim_dir = raptor_pim_dir
@@ -1635,7 +1724,7 @@ def main():
):
compiled_functional = try_stage(
failures,
"Compile Raptor functional PIM",
"Compile Raptor functional Pim",
compile_raptor_target,
functional_model_path,
out_dir / "raptor_functional",
@@ -1659,9 +1748,9 @@ def main():
if wrote_inputs and reference_dirs and outputs_desc:
validation = try_stage(
failures,
"Functional Validation Raptor",
"Functional validation Raptor",
run_functional_validation,
"Functional Validation Raptor",
"Functional validation Raptor",
raptor_functional_pim_dir,
raptor_functional_pim_dir / "config.json",
out_dir / "simulation/out.bin",
@@ -1679,7 +1768,7 @@ def main():
else:
raptor_validation = skipped_validation("Raptor input materialization failed")
elif not reuse_raptor:
raptor_validation = skipped_validation("Raptor PIM compilation did not produce a PIM directory")
raptor_validation = skipped_validation("Raptor Pim compilation did not produce a Pim directory")
pimcomp_model_path = model_path
@@ -1687,7 +1776,7 @@ def main():
reused_pimcomp_dir = args.reuse_pimcomp_dir.resolve()
copied_pimcomp = try_stage_success(
failures,
"Reuse PIMCOMP outputs",
"Reuse Pimcomp outputs",
copy_pimcomp_outputs,
reused_pimcomp_dir,
out_dir / "pimcomp/output",
@@ -1695,12 +1784,12 @@ def main():
if copied_pimcomp:
verification_info = out_dir / "pimcomp/output/VerificationInfo.json"
simulation_info = out_dir / "pimcomp/output/SimulationInfo.gz"
print_step("Reuse PIMCOMP")
print_step("Reuse Pimcomp")
print(f" Directory: {reused_pimcomp_dir}")
elif not reuse_pimcomp:
elif run_pimcomp:
compiled_pimcomp = try_stage(
failures,
"Compile PIMCOMP",
"Compile Pimcomp",
compile_pimcomp,
args,
pimcomp_model_path,
@@ -1710,12 +1799,12 @@ def main():
if compiled_pimcomp is not None:
verification_info, simulation_info = compiled_pimcomp
if reuse_pimcomp:
if reuse_pimcomp or args.raptor_only:
pass
elif verification_info is not None and simulation_info is not None and model_io is not None:
exported = try_stage(
failures,
"Export PIMCOMP for Functional Validation",
"Export Pimcomp for functional validation",
export_pimcomp_for_rust,
pimcomp_model_path,
verification_info,
@@ -1728,14 +1817,14 @@ def main():
elif verification_info is None or simulation_info is None:
record_failure(
failures,
"Export PIMCOMP for Functional Validation",
"PIMCOMP functional export failed because PIMCOMP did not produce VerificationInfo.json and SimulationInfo.gz.",
"Export Pimcomp for functional validation",
"Pimcomp functional export failed because Pimcomp did not produce VerificationInfo.json and SimulationInfo.gz.",
)
else:
record_failure(
failures,
"Export PIMCOMP for Functional Validation",
"PIMCOMP functional export failed because model inputs are not available.",
"Export Pimcomp for functional validation",
"Pimcomp functional export failed because model inputs are not available.",
)
if input_batch is not None and pimcomp_export_dir is not None:
@@ -1745,12 +1834,12 @@ def main():
transform=flatten_pimcomp_input,
)
if not reuse_pimcomp and pimcomp_export_dir is not None and reference_dirs and outputs_desc:
if run_pimcomp and pimcomp_export_dir is not None and reference_dirs and outputs_desc:
validation = try_stage(
failures,
"Functional Validation PIMCOMP",
"Functional validation Pimcomp",
run_functional_validation,
"Functional Validation PIMCOMP",
"Functional validation Pimcomp",
pimcomp_export_dir,
pimcomp_export_dir / "config.json",
out_dir / "simulation/pimcomp.out.bin",
@@ -1761,11 +1850,11 @@ def main():
args,
channel_last=True,
)
pimcomp_validation = validation if validation is not None else failed_validation("PIMCOMP validation failed")
elif reuse_pimcomp:
pimcomp_validation = validation if validation is not None else failed_validation("Pimcomp validation failed")
elif reuse_pimcomp or args.raptor_only:
pass
elif pimcomp_export_dir is None:
pimcomp_validation = failed_validation("PIMCOMP functional export is not available")
pimcomp_validation = failed_validation("Pimcomp functional export is not available")
elif not reference_dirs:
pimcomp_validation = failed_validation("Reference outputs are not available")
else:
@@ -1774,7 +1863,7 @@ def main():
if not args.skip_pimsim_nn and hardware["core_count"] > 0:
written_config = try_stage(
failures,
"Prepare pimsim-nn config",
"Prepare Pimsim config",
prepare_pimsim_config,
args,
hardware,
@@ -1784,25 +1873,25 @@ def main():
elif not args.skip_pimsim_nn:
record_failure(
failures,
"Skip pimsim-nn config",
"pimsim-nn config was skipped because the hardware configuration is not available.",
"Skip Pimsim config",
"Pimsim config was skipped because the hardware configuration is not available.",
)
if args.skip_pimsim_nn:
if not reuse_raptor:
raptor_perf = skipped_perf("Skipped by --skip-pimsim-nn")
if not reuse_pimcomp:
if run_pimcomp:
pimcomp_perf = skipped_perf("Skipped by --skip-pimsim-nn")
elif pimsim_config is None:
if not reuse_raptor:
raptor_perf = skipped_perf("pimsim-nn config is not available")
if not reuse_pimcomp:
pimcomp_perf = skipped_perf("pimsim-nn config is not available")
raptor_perf = skipped_perf("Pimsim config is not available")
if run_pimcomp:
pimcomp_perf = skipped_perf("Pimsim config is not available")
else:
if not reuse_raptor and raptor_pim_dir is not None:
raptor_pimsim_dir = try_stage(
failures,
"Export Raptor for pimsim-nn",
"Export Raptor for Pimsim",
export_raptor_pimsim_artifact,
raptor_pim_dir,
out_dir / "raptor/pimsim_nn",
@@ -1810,25 +1899,25 @@ def main():
if raptor_pimsim_dir is not None:
perf = try_stage(
failures,
"Non-Functional Simulation Raptor",
"Non-functional simulation Raptor",
run_pimsim_nn,
"Non-Functional Simulation Raptor",
"Non-functional simulation Raptor",
raptor_pimsim_dir,
pimsim_config,
steps,
args,
)
raptor_perf = perf if perf is not None else failed_perf("pimsim-nn Raptor failed")
raptor_perf = perf if perf is not None else failed_perf("Pimsim Raptor failed")
else:
raptor_perf = failed_perf("Raptor pimsim-nn export failed")
raptor_perf = failed_perf("Raptor Pimsim export failed")
elif not reuse_raptor:
raptor_perf = skipped_perf("Raptor PIM directory is not available")
raptor_perf = skipped_perf("Raptor Pim directory is not available")
if not reuse_pimcomp:
if run_pimcomp:
if simulation_info is not None:
pimcomp_pimsim_dir = try_stage(
failures,
"Export PIMCOMP for pimsim-nn",
"Export Pimcomp for Pimsim",
export_pimcomp_for_pimsim_nn,
simulation_info,
out_dir / "pimcomp/pimsim_nn",
@@ -1836,32 +1925,32 @@ def main():
if pimcomp_pimsim_dir is not None:
perf = try_stage(
failures,
"Non-Functional Simulation PIMCOMP",
"Non-functional simulation Pimcomp",
run_pimsim_nn,
"Non-Functional Simulation PIMCOMP",
"Non-functional simulation Pimcomp",
pimcomp_pimsim_dir,
pimsim_config,
steps,
args,
)
pimcomp_perf = perf if perf is not None else failed_perf("pimsim-nn PIMCOMP failed")
pimcomp_perf = perf if perf is not None else failed_perf("Pimsim Pimcomp failed")
else:
pimcomp_perf = failed_perf("PIMCOMP pimsim-nn export failed")
pimcomp_perf = failed_perf("Pimcomp Pimsim export failed")
else:
pimcomp_perf = skipped_perf("PIMCOMP SimulationInfo.gz is not available")
pimcomp_perf = skipped_perf("Pimcomp SimulationInfo.gz is not available")
if not reuse_raptor and raptor_pim_dir is not None and raptor_pim_dir.exists():
parsed = try_stage(failures, "Parse Raptor instructions", parse_raptor_instructions, raptor_pim_dir)
raptor_instr = parsed if parsed is not None else empty_instruction_summary(error="Failed to parse Raptor instructions")
elif not reuse_raptor:
raptor_instr = empty_instruction_summary("Raptor PIM directory is not available")
raptor_instr = empty_instruction_summary("Raptor Pim directory is not available")
if not reuse_pimcomp:
if run_pimcomp:
if simulation_info is not None and simulation_info.exists():
parsed = try_stage(failures, "Parse PIMCOMP instructions", parse_pimcomp_instructions, simulation_info)
pimcomp_instr = parsed if parsed is not None else empty_instruction_summary(error="Failed to parse PIMCOMP instructions")
parsed = try_stage(failures, "Parse Pimcomp instructions", parse_pimcomp_instructions, simulation_info)
pimcomp_instr = parsed if parsed is not None else empty_instruction_summary(error="Failed to parse Pimcomp instructions")
else:
pimcomp_instr = empty_instruction_summary("PIMCOMP SimulationInfo.gz is not available")
pimcomp_instr = empty_instruction_summary("Pimcomp SimulationInfo.gz is not available")
report_path = out_dir / "pimcomp/comparison_report.md"
write_report(
@@ -1890,7 +1979,7 @@ def main():
"pimsim_time_ms": args.pimsim_time_ms,
"pimcomp_pipeline": args.pimcomp_pipeline,
"pimcomp_replication": args.pimcomp_replication,
"pimcomp_model_source": "original_onnx",
"pimcomp_model_source": "not_run" if args.raptor_only else "original_onnx",
"pimcomp_config": str(args.pimcomp_config),
"raptor_extra_args": args.raptor_extra_arg,
"reused_raptor_report": optional_path(args.reuse_raptor_report.resolve()) if reuse_raptor else None,
@@ -1926,7 +2015,11 @@ def main():
json.dump(json_report, f, indent=2)
f.write("\n")
partial_side = "pimcomp" if reuse_raptor else "raptor" if args.reuse_pimcomp_report else None
partial_side = (
"pimcomp" if reuse_raptor
else "raptor" if args.raptor_only or args.reuse_pimcomp_report
else None
)
other_side = "RAPTOR" if partial_side == "pimcomp" else "PIMCOMP"
relevant_failures = (
failures if partial_side is None
@@ -1947,7 +2040,7 @@ def main():
for result in relevant_validations
)
failed = bool(relevant_failures or failed_steps or functional_failure)
print("\n" + Style.BRIGHT + Fore.GREEN + "[Completed]" + Style.RESET_ALL)
print(Style.BRIGHT + Fore.GREEN + "[Completed]" + Style.RESET_ALL)
print(f" Report: {report_path}")
print(f" JSON: {json_path}")
if failures or failed_steps:
validation/tools/pim/pimcomp/compare/run_pimcomp_paper_latency.py → validation/tools/pim/pimcomp/compare/run_pimcomp_models.py
+192 -105
View File
@@ -19,33 +19,30 @@ from colorama import Fore, Style
REPO = Path(__file__).resolve().parents[5]
SUITE = REPO / "validation/networks/pimcomp_models"
sys.path.insert(0, str(REPO / "validation"))
from raptor_validation.pimsim_nn import parse_pimsim_nn_metrics # noqa: E402
from raptor_validation.validate_one import STAGE_COLORS # noqa: E402
from raptor_validation.pimcomp_models import (
FUNCTIONAL_MODELS,
MODELS,
SUITE,
add_models_argument,
)
from raptor_validation.artifacts import artifacts_dir, remove_lock_files
from raptor_validation.pimsim_nn import parse_pimsim_nn_metrics
from raptor_validation.validate_one import STAGE_COLORS
PIMCOMP_SOURCE = REPO / "third_party/PIMCOMP-NN"
PIMCOMP_CONFIGS = REPO / "validation/pimsim_configs/pimcomp"
COMPARE = Path(__file__).resolve().with_name("compare_raptor_pimcomp.py")
COMPARE = Path(__file__).resolve().with_name("compare_raptor_pimcomp_model.py")
ARCHES = tuple(sorted(path.name for path in PIMCOMP_CONFIGS.iterdir() if path.is_dir()))
MODELS = {
"vgg8": SUITE / "vgg8/vgg8-mnist-reconstructed.onnx",
"resnet18": SUITE / "resnet18/resnet18-v1-7.onnx",
"resnet34": SUITE / "resnet34/resnet34-v1-7.onnx",
"googlenet": SUITE / "googlenet/googlenet-12-pimsim-nn.onnx",
"yolo11n": SUITE / "yolo11n/yolo11n-pimsim-nn.onnx",
}
FUNCTIONAL_MODELS = {
**MODELS,
"yolo11n": REPO / "validation/networks/yolo11n/depth_51/yolo11n_depth_51.onnx",
}
DEFAULT_ARCHES = ("arch-a", "arch-b")
COMPARISONS = (
("latency", 1, "element"),
("throughput", 2, "batch"),
("throughput", 4, "batch"),
("throughput", 8, "batch"),
)
RESULTS_FILENAME = "results_comparison.csv"
@dataclass(frozen=True)
@@ -67,39 +64,60 @@ def model_dir(root: Path | None, name: str) -> Path:
return root / name if root is not None else MODELS[name].parent
def result_dir(root: Path | None, name: str, arch: str, mode: str, pipeline: int) -> Path:
base = model_dir(root, name)
def result_dir(
root: Path | None,
name: str,
arch: str,
mode: str,
pipeline: int,
ablation_variant: str | None = None,
) -> Path:
base = artifacts_dir(model_dir(root, name))
suffix = "latency" if mode == "latency" else f"throughput/pipeline{pipeline}"
return base / arch / suffix
result = base / arch / suffix
if ablation_variant is not None:
result /= Path("ablation") / ablation_variant
return result
def common_dir(root: Path | None, name: str) -> Path:
def common_dir(root: Path | None, name: str, common_root: Path | None = None) -> Path:
suffix = "common" if FUNCTIONAL_MODELS[name] == MODELS[name] else "common-functional"
return model_dir(root, name) / suffix
base = common_root / name if common_root is not None else model_dir(root, name)
return artifacts_dir(base) / suffix
def clean_artifacts(root: Path | None, models: list[str], arches: list[str]) -> int:
def clean_artifacts(
root: Path | None,
models: list[str],
common_root: Path | None = None,
) -> int:
removed = 0
for name in models:
base = model_dir(root, name)
arch_dirs = {base / arch for arch in arches}
arch_dirs.update(path for path in base.glob("arch-*") if path.is_dir() and not path.is_symlink())
for path in arch_dirs:
for path in (artifacts_dir(base),):
if path.is_dir() and not path.is_symlink():
shutil.rmtree(path)
removed += 1
for common in (base / "common", base / "common-functional"):
if common.is_dir() and not common.is_symlink():
shutil.rmtree(common)
removed += 1
for path in (
(root or SUITE) / "results.csv",
(root or SUITE) / "results_latency.csv",
(root or SUITE) / "results_throughput.csv",
(root or SUITE) / RESULTS_FILENAME,
(root or SUITE) / "results_ablation.csv",
):
if path.is_file() or path.is_symlink():
path.unlink()
removed += 1
summary_root = root or SUITE
if (summary_root / "ablation").is_dir():
for path in (summary_root / "ablation").glob("*/results_comparison.csv"):
path.unlink(missing_ok=True)
removed += 1
removed += remove_lock_files(summary_root)
if common_root is not None:
for name in models:
path = artifacts_dir(common_root / name)
if path.is_dir() and not path.is_symlink():
shutil.rmtree(path)
removed += 1
removed += remove_lock_files(common_root)
return removed
@@ -108,8 +126,11 @@ def write_results_csv(
arch: str,
models: list[str],
comparisons: tuple[tuple[str, int, str], ...] = COMPARISONS,
ablation_variant: str | None = None,
) -> Path:
output = (root or SUITE) / "results.csv"
output = (root or SUITE) / RESULTS_FILENAME
if ablation_variant is not None:
output = output.parent / "ablation" / ablation_variant / RESULTS_FILENAME
fields = (
"arch",
"model",
@@ -152,7 +173,9 @@ def write_results_csv(
rows.append({field: row.get(field, "NA") for field in fields})
for name in models:
for comparison_mode, pipeline, pimcomp_pipeline in comparisons:
report_path = result_dir(root, name, arch, comparison_mode, pipeline) / "pimcomp/comparison_report.json"
report_path = result_dir(
root, name, arch, comparison_mode, pipeline, ablation_variant
) / "pimcomp/comparison_report.json"
row = {
"model": name,
"arch": arch,
@@ -215,6 +238,7 @@ def write_results_csv(
int(row["raptor_pipeline"]),
)
)
output.parent.mkdir(parents=True, exist_ok=True)
with open(output, "w", newline="", encoding="utf-8") as f:
writer = csv.DictWriter(f, fieldnames=fields, lineterminator="\n")
writer.writeheader()
@@ -275,11 +299,11 @@ def format_value(value: float | None) -> str:
def print_stage(title: str, color: str) -> None:
print("\n" + Style.BRIGHT + color + f"[{title}]" + Style.RESET_ALL, flush=True)
print(Style.BRIGHT + color + f"[{title}]" + Style.RESET_ALL, flush=True)
def print_completed(label: str, output: str = "") -> None:
print("\n" + Style.BRIGHT + Fore.CYAN + f"[Completed {label}]" + Style.RESET_ALL)
print(Style.BRIGHT + Fore.CYAN + f"[Completed {label}]" + Style.RESET_ALL)
if output:
print(output, end="" if output.endswith("\n") else "\n")
print("=" * 72, flush=True)
@@ -321,7 +345,7 @@ def validate_pimcomp_source() -> None:
source = header.read_text(encoding="utf-8")
for setting in ("int population_num = 200;", "int max_iteration = 1000;"):
if setting not in source:
raise RuntimeError(f"PIMCOMP paper setting is missing: {setting}")
raise RuntimeError(f"Pimcomp paper setting is missing: {setting}")
def comparison_command(
@@ -337,6 +361,8 @@ def comparison_command(
batch_size: int,
timeout: float,
fast: bool,
raptor_extra_args: list[str] | tuple[str, ...] = (),
raptor_only: bool = False,
reuse_raptor_report: Path | None = None,
reuse_pimcomp_dir: Path | None = None,
reuse_pimcomp_report: Path | None = None,
@@ -373,7 +399,9 @@ def comparison_command(
pimcomp_pipeline,
"--pimcomp-replication",
"GA",
*(["--raptor-only"] if raptor_only else []),
f"--raptor-extra-arg=--pipeline={pipeline}",
*[f"--raptor-extra-arg={arg}" for arg in raptor_extra_args],
"--timeout-seconds",
str(timeout),
"--fail-on-error",
@@ -418,13 +446,19 @@ def comparison_command_for(
args.batch_size,
args.timeout_seconds,
not args.no_fast,
args.raptor_extra_args,
args.raptor_only,
reuse_raptor_report=(
report
if args.only == "pimcomp"
else None
),
reuse_pimcomp_dir=spec.shared_pimcomp_dir if reuse_shared_pimcomp and args.only != "raptor" else None,
reuse_pimcomp_report=report if args.only == "raptor" else None,
reuse_pimcomp_dir=(
spec.shared_pimcomp_dir
if reuse_shared_pimcomp and args.only != "raptor" and not args.raptor_only
else None
),
reuse_pimcomp_report=report if args.only == "raptor" and not args.raptor_only else None,
)
@@ -497,20 +531,29 @@ def config_path(arch: str, mode: str, sim_time_ms: int, *, write: bool) -> Path:
def main() -> int:
parser = argparse.ArgumentParser(
description="Compare supported PIMCOMP models with Raptor latency and throughput schedules."
description="Compare supported Pimcomp models with Raptor latency and throughput schedules."
)
parser.add_argument(
"--out-dir",
type=Path,
help="Result root (default: artifacts beside each model under validation/).",
help="Suite root; generated artifacts go below each model's artifacts/ directory (default: validation/networks/pimcomp_models).",
)
parser.add_argument("--models", nargs="+", choices=MODELS, default=list(MODELS))
parser.add_argument(
"--arch",
"--common-dir",
type=Path,
help="Shared root for per-model reference artifacts. Default: inside --out-dir.",
)
parser.add_argument(
"--ablation-variant",
help="Place Raptor artifacts below <mode>/ablation/<variant> and write the comparison summary below ablation/<variant>.",
)
add_models_argument(parser)
parser.add_argument(
"--archs",
nargs="+",
choices=ARCHES,
default=list(ARCHES),
help="PIM architectures to run (default: all architectures).",
default=list(DEFAULT_ARCHES),
help=f"Pim architectures to run (default: {', '.join(DEFAULT_ARCHES)}).",
)
parser.add_argument(
"--mode",
@@ -524,6 +567,11 @@ def main() -> int:
choices=("raptor", "pimcomp"),
help="Re-run only this compiler's compile, validation, and simulation stages; preserve the other side from its report.",
)
parser.add_argument(
"--raptor-only",
action="store_true",
help="Run only Raptor; do not compile, validate, or simulate Pimcomp. Default: off.",
)
parser.add_argument(
"--pipeline",
type=int,
@@ -534,7 +582,7 @@ def main() -> int:
"--pimsim-time-ms",
type=int,
default=1000,
help="throughput pimsim-nn convergence deadline in ms (default: 1000).",
help="Throughput Pimsim convergence deadline in ms (default: 1000).",
)
parser.add_argument(
"--batch-size",
@@ -564,15 +612,32 @@ def main() -> int:
parser.add_argument(
"--no-fast",
action="store_true",
help="Disable fast pimsim-nn throughput convergence for authoritative experiments.",
help="Disable fast Pimsim throughput convergence for authoritative experiments.",
)
parser.add_argument(
"--raptor-extra-arg",
action="append",
default=[],
dest="raptor_extra_args",
help="Additional argument to pass to Raptor; repeat as needed.",
)
args = parser.parse_args()
if args.ablation_variant and Path(args.ablation_variant).name != args.ablation_variant:
parser.error("--ablation-variant must be a single directory name")
if args.only is not None and args.raptor_only:
parser.error("--only cannot be combined with --raptor-only")
out_dir = args.out_dir.resolve() if args.out_dir is not None else None
args.arch = list(dict.fromkeys(args.arch))
common_root = args.common_dir.resolve() if args.common_dir is not None else None
args.archs = list(dict.fromkeys(args.archs))
args.mode = list(dict.fromkeys(args.mode))
if args.clean:
print(f"Removed {clean_artifacts(out_dir, args.models, args.arch)} comparison artifact path(s).")
print(
f"Removed {clean_artifacts(out_dir, args.models, common_root)} "
"comparison artifact path(s)."
)
return 0
if args.jobs < 1:
@@ -585,7 +650,7 @@ def main() -> int:
parser.error("--timeout-seconds must be non-negative")
comparisons_by_arch: dict[str, tuple[tuple[str, int, str], ...]] = {}
configs_by_arch: dict[str, dict[str, Path]] = {}
for arch in args.arch:
for arch in args.archs:
comparisons = tuple(
comparison for comparison in COMPARISONS
if comparison[0] in args.mode
@@ -611,10 +676,12 @@ def main() -> int:
if args.only is not None:
missing_reuse = []
for arch in args.arch:
for arch in args.archs:
for name in args.models:
for mode, pipeline, _ in comparisons_by_arch[arch]:
comparison_dir = result_dir(out_dir, name, arch, mode, pipeline)
comparison_dir = result_dir(
out_dir, name, arch, mode, pipeline, args.ablation_variant
)
required = comparison_dir / "pimcomp/comparison_report.json"
if not required.exists():
missing_reuse.append(str(required))
@@ -629,14 +696,15 @@ def main() -> int:
+ ", ".join(missing_reuse)
)
validate_pimcomp_source()
if not args.raptor_only:
validate_pimcomp_source()
if out_dir is not None and not args.dry_run:
out_dir.mkdir(parents=True, exist_ok=True)
print(Style.BRIGHT + f"Found {len(args.models)} PIMCOMP model(s) to compare." + Style.RESET_ALL)
print(f"Architectures: {', '.join(args.arch)}")
print(Style.BRIGHT + f"Found {len(args.models)} Pimcomp model(s) to compare." + Style.RESET_ALL)
print(f"Architectures: {', '.join(args.archs)}")
print(f"Modes: {', '.join(args.mode)}")
print(f"Throughput pimsim time: {args.pimsim_time_ms} ms")
print(f"Throughput Pimsim time: {args.pimsim_time_ms} ms")
print(f"Max parallel jobs: {args.jobs}")
print(
f"Comparison jobs: "
@@ -645,29 +713,36 @@ def main() -> int:
print(f"Results root: {out_dir or SUITE}")
print("=" * 72)
print_stage("Prepare shared artifacts", STAGE_COLORS["Build Runner"])
for name in args.models:
try:
run(
prepare_common_command(
FUNCTIONAL_MODELS[name],
common_dir(out_dir, name),
args.timeout_seconds,
),
dry_run=args.dry_run,
)
finally:
print_completed(f"shared artifacts: {name}")
print_stage("Prepare shared artifacts", STAGE_COLORS["Build runner"])
try:
for name in args.models:
try:
run(
prepare_common_command(
FUNCTIONAL_MODELS[name],
common_dir(out_dir, name, common_root),
args.timeout_seconds,
),
dry_run=args.dry_run,
)
finally:
print_completed(f"shared artifacts: {name}")
except KeyboardInterrupt:
remove_lock_files(out_dir or SUITE)
print("Interrupted; cleaned validation lock files.", file=sys.stderr)
return 130
failed = []
comparison_specs: list[ComparisonSpec] = []
shared_pimcomp_by_group: dict[tuple[str, str, str], Path] = {}
for arch in args.arch:
for arch in args.archs:
comparisons = comparisons_by_arch[arch]
configs = configs_by_arch[arch]
for name in args.models:
for mode, pipeline, pimcomp_pipeline in comparisons:
model_result_dir = result_dir(out_dir, name, arch, mode, pipeline)
model_result_dir = result_dir(
out_dir, name, arch, mode, pipeline, args.ablation_variant
)
label = f"{arch}/{name}/{mode}/pipeline{pipeline}"
group = (arch, name, mode)
shared_pimcomp_dir = shared_pimcomp_by_group.get(group)
@@ -681,7 +756,7 @@ def main() -> int:
model=MODELS[name],
functional_model=FUNCTIONAL_MODELS[name],
output_dir=model_result_dir,
common_dir=common_dir(out_dir, name),
common_dir=common_dir(out_dir, name, common_root),
config=configs[mode],
mode=mode,
pipeline=pipeline,
@@ -701,7 +776,8 @@ def main() -> int:
if run(command, dry_run=True, check=False):
failed.append(spec.label)
elif comparison_specs:
anchor_specs = comparison_specs if args.only == "raptor" else [
raptor_only_run = args.only == "raptor" or args.raptor_only
anchor_specs = comparison_specs if raptor_only_run else [
spec for spec in comparison_specs if spec.anchor
]
anchor_jobs = [
@@ -712,64 +788,75 @@ def main() -> int:
)
for spec in anchor_specs
]
dependent_specs = [] if args.only == "raptor" else [
dependent_specs = [] if raptor_only_run else [
spec for spec in comparison_specs if not spec.anchor
]
print_directly = min(args.jobs, len(comparison_specs)) == 1
with (nullcontext(None) if print_directly else TemporaryDirectory(prefix="raptor-pimcomp-")) as log_dir:
anchor_failed, log_offset = run_comparison_jobs(
anchor_jobs,
args.jobs,
Path(log_dir) if log_dir else None,
0,
)
failed.extend(anchor_failed)
dependent_jobs = [
(
spec.label,
comparison_command_for(
spec,
args,
reuse_shared_pimcomp=pimcomp_artifact_ready(spec.shared_pimcomp_dir),
),
None,
try:
with (nullcontext(None) if print_directly else TemporaryDirectory(prefix="raptor-pimcomp-")) as log_dir:
anchor_failed, log_offset = run_comparison_jobs(
anchor_jobs,
args.jobs,
Path(log_dir) if log_dir else None,
0,
)
for spec in dependent_specs
]
dependent_failed, _ = run_comparison_jobs(
dependent_jobs,
args.jobs,
Path(log_dir) if log_dir else None,
log_offset,
)
failed.extend(dependent_failed)
failed.extend(anchor_failed)
dependent_jobs = [
(
spec.label,
comparison_command_for(
spec,
args,
reuse_shared_pimcomp=pimcomp_artifact_ready(spec.shared_pimcomp_dir),
),
None,
)
for spec in dependent_specs
]
dependent_failed, _ = run_comparison_jobs(
dependent_jobs,
args.jobs,
Path(log_dir) if log_dir else None,
log_offset,
)
failed.extend(dependent_failed)
except KeyboardInterrupt:
remove_lock_files(out_dir or SUITE)
print("Interrupted; cleaned validation lock files.", file=sys.stderr)
return 130
if args.dry_run:
return 1 if failed else 0
remove_lock_files(out_dir or SUITE)
results_path = None
for arch in args.arch:
for arch in args.archs:
results_path = write_results_csv(
out_dir,
arch,
args.models,
comparisons_by_arch[arch],
args.ablation_variant,
)
assert results_path is not None
print_stage(results_path.name, STAGE_COLORS["Compare Outputs"])
print_stage(results_path.name, STAGE_COLORS["Compare outputs"])
print(results_path.read_text(encoding="utf-8"), end="")
for arch in args.arch:
for arch in args.archs:
for name in args.models:
for mode, pipeline, _ in comparisons_by_arch[arch]:
label = f"{arch}/{name}/{mode}/pipeline{pipeline}"
report_path = result_dir(out_dir, name, arch, mode, pipeline) / "pimcomp/comparison_report.json"
report_path = result_dir(
out_dir, name, arch, mode, pipeline, args.ablation_variant
) / "pimcomp/comparison_report.json"
compiler = "raptor" if args.raptor_only else args.only
if not report_path.exists() or not comparison_passed(
json.loads(report_path.read_text(encoding="utf-8")),
args.only,
compiler,
):
if label not in failed:
failed.append(label)
print("\n" + Style.BRIGHT + Fore.CYAN + "Summary" + Style.RESET_ALL)
print(Style.BRIGHT + Fore.CYAN + "Summary" + Style.RESET_ALL)
total_jobs = sum(len(args.models) * len(comparisons) for comparisons in comparisons_by_arch.values())
print(Style.BRIGHT + f"Passed: {total_jobs - len(failed)}" + Style.RESET_ALL)
print(Style.BRIGHT + f"Failed: {len(failed)}" + Style.RESET_ALL)
validation/tools/pim/pimcomp/correctness/README.md
+46 -10
View File
@@ -1,6 +1,6 @@
# PIMCOMP batch correctness reproduction
# Pimcomp batch correctness reproduction
PIMCOMP's batch scheduler currently emits an incomplete standalone program for
Pimcomp's batch scheduler currently emits an incomplete standalone program for
models containing post operations. The generated `VerificationInfo.json` uses a
negative `source_address` to identify the preceding node, but the batch
verifier resolves that address by copying the provider tensor directly from
@@ -17,8 +17,8 @@ the missing computation visible in Rust functional validation.
For the checked-in GoogLeNet throughput/pipeline2 artifact, 39 provider tensors
are referenced by batch loads. Nineteen have generated stores; twenty are
never written. Preloading the provider tensors with the same ONNX Runtime
intermediates used by PIMCOMP's verifier makes the exported program pass. This
reproduces the verifier's input contract; it does not repair PIMCOMP's batch
intermediates used by Pimcomp's verifier makes the exported program pass. This
reproduces the verifier's input contract; it does not repair Pimcomp's batch
schedule.
Run the default reproduction from the repository root:
@@ -27,23 +27,59 @@ Run the default reproduction from the repository root:
.venv/bin/python validation/tools/pim/pimcomp/correctness/run_prefill_experiment.py
```
The launcher accepts an alternate comparison directory, model, and work
work directory, and shared reference-artifact directory:
The launcher accepts an alternate comparison directory, model, work directory,
and shared reference-artifact directory:
```bash
.venv/bin/python validation/tools/pim/pimcomp/correctness/run_prefill_experiment.py \
validation/networks/pimcomp_models/googlenet/arch-a/throughput/pipeline2 \
validation/networks/pimcomp_models/googlenet/artifacts/arch-a/throughput/pipeline2 \
validation/networks/pimcomp_models/googlenet/googlenet-12-pimsim-nn.onnx \
/tmp/pimcomp-prefill-googlenet \
validation/networks/pimcomp_models/googlenet/common
validation/networks/pimcomp_models/googlenet/artifacts/arch-a/throughput/pipeline2/correctness/prefill \
validation/networks/pimcomp_models/googlenet/artifacts/common
```
Without the optional work-directory argument, the experiment uses the same
`correctness/prefill/` directory below the comparison artifacts.
It runs the exported artifact once with its original memory image and once
with [`prefill_batch_memory.py`](prefill_batch_memory.py), then compares both
outputs with the recorded native reference. The expected GoogLeNet result is a
baseline maximum difference near `6.70705` and a prefilled maximum difference
near `4.05e-6`.
The issue is in PIMCOMP batch scheduling/validation semantics, not in the Rust
The issue is in Pimcomp batch scheduling/validation semantics, not in the Rust
simulator's vector-length interpretation. Vector lengths remain element counts
as specified by the reference ISA.
## ResNet BatchNorm correctness gap
Pimcomp has a separate correctness limitation in its ResNet element pipeline.
`BatchNormalization` is an ONNX operation. The frontend's
[`fuse_operators()` pass](../../../../../third_party/PIMCOMP-NN/frontend/frontend.py#L535)
marks it as fused and removes the node from the
scheduled graph, but the released path does not fold the BatchNorm affine
transform into the convolution weights and bias. The corresponding verifier
workaround in [`verification.py`](../../../../../third_party/PIMCOMP-NN/verification/verification.py#L99)
replaces BatchNorm parameters with identity values (scale and variance equal
to one, bias and mean equal to zero) before running ONNX Runtime.
Therefore Pimcomp's native verifier and exported element program agree with
each other, but they do not implement the original ResNet ONNX model. On the
current ResNet-18 Arch-A latency artifact, using the same input:
| Comparison | Maximum absolute difference |
|---|---:|
| Pimcomp native verifier vs Rust export | `6.7e-8` |
| Rust export vs original ONNX reference | `4.8294563` |
The Pimcomp output ranges from approximately `-0.187` to `0.220`, while the
original ONNX output ranges from `-3.572` to `4.834`; 462 of 1000 final
elements differ by more than one. This is not a Rust simulator or Python
exporter regression. It is a Pimcomp model-semantics mismatch caused by
dropping BatchNorm numerics. The same issue affects the ResNet-34 latency
artifact. VGG and GoogLeNet do not show this particular mismatch because they
do not contain the same ResNet BatchNorm path.
The latency comparison intentionally uses the original ONNX reference, so
these Pimcomp rows must remain `FAIL` until Pimcomp folds BatchNorm correctly
or the comparison explicitly uses a BatchNorm-neutralized reference.
validation/tools/pim/pimcomp/correctness/prefill_batch_memory.py
+3 -3
View File
@@ -1,5 +1,5 @@
#!/usr/bin/env python3
"""Populate the host-side intermediate buffers expected by PIMCOMP batch mode."""
"""Populate the host-side intermediate buffers expected by Pimcomp batch mode."""
from __future__ import annotations
@@ -23,7 +23,7 @@ def flatten_reference(value: np.ndarray) -> np.ndarray:
elif value.ndim == 2:
value = value.transpose()
else:
raise ValueError(f"PIMCOMP batch verification only flattens 2D/4D tensors, got {value.shape}")
raise ValueError(f"Pimcomp batch verification only flattens 2D/4D tensors, got {value.shape}")
return value.astype(np.float32, copy=False).reshape(-1)
@@ -94,7 +94,7 @@ def prefill_batch_memory(
session = ort.InferenceSession(runtime_model.SerializeToString(), providers=["CPUExecutionProvider"])
session_inputs = session.get_inputs()
if len(session_inputs) != 1:
raise ValueError("PIMCOMP export currently requires exactly one runtime input tensor")
raise ValueError("Pimcomp export currently requires exactly one runtime input tensor")
input_meta = session_inputs[0]
input_tensor = np.loadtxt(input_path, delimiter=",", dtype=np.float32).reshape(input_meta.shape)
provider_names = [node_list[index]["name"] for index in providers]
validation/tools/pim/pimcomp/correctness/run_prefill_experiment.py
+21 -8
View File
@@ -1,5 +1,5 @@
#!/usr/bin/env python3
"""Reproduce the PIMCOMP batch prefill correctness experiment."""
"""Reproduce the Pimcomp batch prefill correctness experiment."""
from __future__ import annotations
@@ -7,7 +7,6 @@ import argparse
import json
import subprocess
import sys
import tempfile
from pathlib import Path
import numpy as np
@@ -17,9 +16,9 @@ from prefill_batch_memory import prefill_batch_memory
SCRIPT_DIR = Path(__file__).resolve().parent
REPO_ROOT = SCRIPT_DIR.parents[4]
DEFAULT_COMPARISON_DIR = REPO_ROOT / "validation/networks/pimcomp_models/googlenet/arch-a/throughput/pipeline2"
DEFAULT_COMPARISON_DIR = REPO_ROOT / "validation/networks/pimcomp_models/googlenet/artifacts/arch-a/throughput/pipeline2"
DEFAULT_MODEL = REPO_ROOT / "validation/networks/pimcomp_models/googlenet/googlenet-12-pimsim-nn.onnx"
DEFAULT_COMMON_DIR = REPO_ROOT / "validation/networks/pimcomp_models/googlenet/common"
DEFAULT_COMMON_DIR = REPO_ROOT / "validation/networks/pimcomp_models/googlenet/artifacts/common"
SIMULATOR_MANIFEST = REPO_ROOT / "backend-simulators/pim/pim-simulator/Cargo.toml"
@@ -28,6 +27,7 @@ def run_simulator(
memory: Path,
output: Path,
dump: str,
input_dir: Path,
) -> None:
subprocess.run(
[
@@ -44,6 +44,8 @@ def run_simulator(
str(comparison_dir / "pimcomp/exported"),
"--memory",
str(memory),
"--input-dir",
str(input_dir),
"-o",
str(output),
"-d",
@@ -86,14 +88,18 @@ def main() -> int:
model = args.model.resolve()
common_dir = args.common_dir.resolve()
if args.work_dir is None:
work_dir = Path(tempfile.mkdtemp(prefix="pimcomp-prefill."))
work_dir = comparison_dir / "correctness/prefill"
work_dir.mkdir(parents=True, exist_ok=True)
else:
work_dir = args.work_dir.resolve()
work_dir.mkdir(parents=True, exist_ok=True)
input_path = common_dir / "inputs/in0.csv"
if not input_path.is_file():
input_path = comparison_dir / "inputs/in0.csv"
simulator_input_dir = work_dir / "inputs"
simulator_input_dir.mkdir(parents=True, exist_ok=True)
np.loadtxt(input_path, delimiter=",", dtype=np.float32).tofile(
simulator_input_dir / "input_0.bin"
)
prefilled_memory = work_dir / "prefilled_memory.bin"
metadata_path = work_dir / "metadata.json"
@@ -117,8 +123,15 @@ def main() -> int:
comparison_dir / "pimcomp/exported/memory.bin",
baseline_output,
dump,
simulator_input_dir,
)
run_simulator(
comparison_dir,
prefilled_memory,
prefilled_output,
dump,
simulator_input_dir,
)
run_simulator(comparison_dir, prefilled_memory, prefilled_output, dump)
compare_outputs(baseline_output, prefilled_output, reference, work_dir)
return 0
validation/tools/pim/visuals/pim_sequence_diagram.py
+3 -3
View File
@@ -98,9 +98,9 @@ def read_binary(path: Path) -> Program:
magic, version, count = HEADER.unpack_from(data)
expected_size = HEADER.size + count * RECORD.size
if magic != b"PIMB":
raise ValueError(f"{path}: invalid PIM binary magic")
raise ValueError(f"{path}: invalid Pim binary magic")
if version != 1:
raise ValueError(f"{path}: unsupported PIM binary version {version}")
raise ValueError(f"{path}: unsupported Pim binary version {version}")
if len(data) != expected_size:
raise ValueError(f"{path}: expected {expected_size} bytes, found {len(data)}")
@@ -399,7 +399,7 @@ def render_text(cores: list[int], transfers: list[Transfer], programs: dict[int,
def main() -> int:
parser = argparse.ArgumentParser(
description="Generate SequenceDiagram.org text for PIM communication and intervening work."
description="Generate SequenceDiagram.org text for Pim communication and intervening work."
)
parser.add_argument("pim_dir", type=Path, help="Directory containing core_<id>.json or core_<id>.pim files")
selection = parser.add_mutually_exclusive_group(required=True)
validation/tools/yolo_local_image_validation.py
+19 -15
View File
@@ -22,6 +22,7 @@ if sys.version_info < (3, 10):
)
from raptor_validation.onnx_utils import _ONNX_TO_NP, onnx_io, write_inputs_binary, write_inputs_to_memory_bin
from raptor_validation.artifacts import artifacts_dir
from raptor_validation.validate_one import (
MODE_COMPILE_ONLY,
build_dump_ranges,
@@ -64,16 +65,17 @@ def find_network_onnx(network_dir: Path) -> Path:
def local_case_paths(network_dir: Path, case_name: str):
artifact_root = artifacts_dir(network_dir)
return {
"root": network_dir,
"runner": network_dir / "runner" / "build" / "runner",
"runner_build": network_dir / "runner" / "build",
"raptor_pim": network_dir / "raptor" / "pim",
"real_root": network_dir / "real_image_validation",
"input_csv": network_dir / "real_image_validation" / "inputs" / f"{case_name}.csv",
"ref_dir": network_dir / "real_image_validation" / "reference" / case_name,
"sim_dir": network_dir / "real_image_validation" / "simulation" / case_name,
"sim_bin": network_dir / "real_image_validation" / "simulation" / case_name / "out.bin",
"root": artifact_root,
"runner": artifact_root / "runner" / "build" / "runner",
"runner_build": artifact_root / "runner" / "build",
"raptor_pim": artifact_root / "raptor" / "pim",
"real_root": artifact_root / "real_image_validation",
"input_csv": artifact_root / "real_image_validation" / "inputs" / f"{case_name}.csv",
"ref_dir": artifact_root / "real_image_validation" / "reference" / case_name,
"sim_dir": artifact_root / "real_image_validation" / "simulation" / case_name,
"sim_bin": artifact_root / "real_image_validation" / "simulation" / case_name / "out.bin",
}
@@ -102,10 +104,11 @@ def ensure_local_artifacts(args, network_onnx_path: Path):
def ensure_existing_artifacts(network_dir: Path):
artifact_root = artifacts_dir(network_dir)
required_paths = [
network_dir / "runner" / "build" / "runner",
network_dir / "raptor" / "pim" / "config.json",
network_dir / "raptor" / "pim" / "memory.bin",
artifact_root / "runner" / "build" / "runner",
artifact_root / "raptor" / "pim" / "config.json",
artifact_root / "raptor" / "pim" / "memory.bin",
]
missing = [str(path) for path in required_paths if not path.exists()]
if missing:
@@ -137,7 +140,7 @@ def run_local_reference_and_simulator(args, network_dir: Path, network_onnx_path
tensor = np.loadtxt(paths["input_csv"], delimiter=",", dtype=np.float32).reshape(1, 3, 640, 640)
write_inputs_to_memory_bin(paths["raptor_pim"] / "memory.bin", paths["raptor_pim"] / "config.json", [tensor])
input_bin = paths["sim_dir"] / "input.bin"
input_bin = paths["sim_dir"] / "input_0.bin"
write_inputs_binary(input_bin, [tensor])
dump_ranges = build_dump_ranges(paths["raptor_pim"] / "config.json", output_descriptors)
@@ -147,7 +150,8 @@ def run_local_reference_and_simulator(args, network_dir: Path, network_onnx_path
paths["sim_bin"],
dump_ranges,
timeout_sec=args.command_timeout_seconds,
input_paths=[input_bin],
input_dir=input_bin.parent,
batch_size=1,
)
return paths, output_descriptors[0]
@@ -225,7 +229,7 @@ def main():
parser.add_argument(
"--annotated-dir",
type=Path,
default=defaults["network_dir"] / "real_image_validation" / "annotated",
default=defaults["network_dir"] / "artifacts" / "real_image_validation" / "annotated",
)
args = parser.parse_args()
validation/tools/yolo_real_image_validation.py
+4 -4
View File
@@ -262,7 +262,7 @@ def ensure_remote_artifacts(args):
def remote_case_paths(args, case_name: str):
network_dir = Path(args.network_dir)
root = Path(args.remote_project) / network_dir
root = Path(args.remote_project) / network_dir / "artifacts"
return {
"root": root,
"runner": root / "runner" / "build" / "runner",
@@ -272,7 +272,7 @@ def remote_case_paths(args, case_name: str):
"input_csv": root / "real_image_validation" / "inputs" / f"{case_name}.csv",
"ref_dir": root / "real_image_validation" / "reference" / case_name,
"sim_dir": root / "real_image_validation" / "simulation" / case_name,
"sim_input": root / "real_image_validation" / "simulation" / case_name / "input.bin",
"sim_input": root / "real_image_validation" / "simulation" / case_name / "input_0.bin",
"sim_bin": root / "real_image_validation" / "simulation" / case_name / "out.bin",
}
@@ -331,7 +331,7 @@ PY
f"cd {quoted_project}/backend-simulators/pim/pim-simulator && "
f"cargo run --no-default-features --release --package pim-simulator --bin pim-simulator -- "
f"-f {quoted_pim} -o {quoted_sim_bin} -d {dump_range} "
f"--mode latency --input {shlex.quote(str(paths['sim_input']))}"
f"--mode latency --batch-size 1 --input-dir {shlex.quote(str(paths['sim_input'].parent))}"
)
remote_bash(args.ssh_key, args.remote_host, sim_command)
return paths
@@ -408,7 +408,7 @@ def main():
parser.add_argument("--core-count", type=int, default=144)
parser.add_argument("--command-timeout-seconds", type=int, default=7200)
parser.add_argument("--skip-compile", action="store_true")
parser.add_argument("--annotated-dir", default="validation/networks/yolo11n/depth_51/real_image_validation/annotated")
parser.add_argument("--annotated-dir", default="validation/networks/yolo11n/depth_51/artifacts/real_image_validation/annotated")
args = parser.parse_args()
args.ssh_key = str(Path(args.ssh_key).expanduser())