diff --git a/README.md b/README.md
index 94c88cb..4fa7769 100644
--- a/README.md
+++ b/README.md
@@ -73,6 +73,9 @@ differ, and validation defaults to Arch-A.
## Documentation
+The [paper artifact](artifact/README.md) builds the pinned compiler and simulators
+in Docker, runs the comparison and ablation experiments, and generates plots.
+
Start with the [documentation index](docs/README.md). It links the
[compiler architecture](docs/compiler/architecture.md),
[compiler options](docs/compiler/options.md), and
diff --git a/artifact/.dockerignore b/artifact/.dockerignore
new file mode 100644
index 0000000..2160ec4
--- /dev/null
+++ b/artifact/.dockerignore
@@ -0,0 +1,13 @@
+*
+!Dockerfile
+!pins.env
+!requirements.lock
+!models.sha256
+!build.sh
+!run.py
+!plot_ablation.py
+!source.tar.gz
+!source.sha256
+!validation-overlay/
+!validation-overlay/**
+!validation-overlay.sha256
diff --git a/artifact/.gitignore b/artifact/.gitignore
new file mode 100644
index 0000000..7afad21
--- /dev/null
+++ b/artifact/.gitignore
@@ -0,0 +1,7 @@
+/source.tar.gz
+/source.tar.gz.tmp
+/source.sha256
+/dist/
+/results/
+/validation-overlay/
+/validation-overlay.sha256
diff --git a/artifact/Dockerfile b/artifact/Dockerfile
new file mode 100644
index 0000000..1d3b070
--- /dev/null
+++ b/artifact/Dockerfile
@@ -0,0 +1,46 @@
+FROM python:3.12.12-slim-trixie AS python-deps
+
+ENV DEBIAN_FRONTEND=noninteractive \
+ CC=clang-19 CXX=clang++-19 \
+ LDFLAGS=-fuse-ld=mold \
+ RUSTFLAGS="-C linker=clang-19 -C link-arg=-fuse-ld=mold" \
+ CARGO_HOME=/opt/cargo RUSTUP_HOME=/opt/rustup \
+ PATH=/opt/raptor/.venv/bin:/opt/cargo/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
+ OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 MPLBACKEND=Agg \
+ PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 \
+ HOME=/tmp XDG_CACHE_HOME=/tmp/cache MPLCONFIGDIR=/tmp/matplotlib
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+ build-essential ca-certificates clang-19 curl git libomp-19-dev \
+ libzstd-dev mold ninja-build pkg-config zlib1g-dev \
+ && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /opt/artifact
+COPY pins.env requirements.lock models.sha256 build.sh ./
+RUN python -m venv /opt/raptor/.venv \
+ && /opt/raptor/.venv/bin/python -m pip install --no-cache-dir --only-binary=onnx -r requirements.lock
+
+FROM python-deps AS artifact
+ARG BUILD_JOBS
+RUN . ./pins.env && curl --proto '=https' --tlsv1.2 -fsSL https://sh.rustup.rs -o /tmp/rustup.sh \
+ && sh /tmp/rustup.sh -y --profile minimal --default-toolchain "$RUST_VERSION" \
+ && rm /tmp/rustup.sh
+
+COPY source.tar.gz source.sha256 ./
+RUN sha256sum -c source.sha256 \
+ && . ./pins.env \
+ && gzip -dc source.tar.gz > /tmp/raptor-source.tar \
+ && test "$(git get-tar-commit-id < /tmp/raptor-source.tar)" = "$RAPTOR_COMMIT" \
+ && tar -xf /tmp/raptor-source.tar -C /opt/raptor \
+ && rm source.tar.gz /tmp/raptor-source.tar
+RUN bash build.sh "${BUILD_JOBS:-$(env -u OMP_NUM_THREADS -u OMP_THREAD_LIMIT nproc)}"
+COPY validation-overlay/ ./validation-overlay/
+COPY validation-overlay.sha256 ./
+RUN sha256sum -c validation-overlay.sha256 \
+ && cp -R validation-overlay/validation/. /opt/raptor/validation/ \
+ && cp validation-overlay.sha256 /opt/artifact/provenance/
+COPY run.py plot_ablation.py ./
+
+WORKDIR /opt/raptor
+ENTRYPOINT ["/opt/raptor/.venv/bin/python", "/opt/artifact/run.py"]
+CMD ["reproduce"]
diff --git a/artifact/README.md b/artifact/README.md
new file mode 100644
index 0000000..978b1d9
--- /dev/null
+++ b/artifact/README.md
@@ -0,0 +1,32 @@
+# Raptor paper artifact
+
+Raptor is fixed to commit `4a1fe710b2f0e4e9c0504f4fd9b4c4dd3df53b7d`.
+
+Requires Docker on Linux or WSL2. The build downloads dependencies and uses Clang 19 and mold.
+Install the host dependencies:
+
+```bash
+sudo apt install -y docker.io docker-cli docker-buildx git
+sudo systemctl enable --now docker
+sudo usermod -aG docker "$USER"
+newgrp docker
+```
+
+Log out and back in, then run from this directory:
+
+```bash
+bash artifact.sh build
+bash artifact.sh reproduce
+```
+
+Open `results/full/index.html` to view the plots and result tables.
+The page also links to the CSV files and logs. Failed numerical validation remains visible in the results.
+
+For a small VGG8 run, use `bash artifact.sh smoke` and open `results/smoke/index.html`.
+Build jobs, run jobs, and threads per job default to the available logical processor count.
+Override with `BUILD_JOBS`, `JOBS`, and `THREADS_PER_JOB`, or run `bash artifact.sh reproduce --jobs 8 --threads-per-job 4`.
+For another run, set `OUTPUT_DIR` to a new directory.
+
+To prepare the paper attachment, run `bash artifact/artifact.sh package` from the repository root.
+The archive is written to `artifact/dist/` and includes the pinned sources and models.
+Updated scheduling code is included separately; compiler and simulator sources stay pinned.
diff --git a/artifact/artifact.sh b/artifact/artifact.sh
new file mode 100755
index 0000000..257f0e5
--- /dev/null
+++ b/artifact/artifact.sh
@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+set -euo pipefail
+cd -- "$(dirname -- "${BASH_SOURCE[0]}")"
+source pins.env
+image="raptor-artifact:${RAPTOR_COMMIT:0:12}"
+command=${1:-help}
+if (($#)); then shift; fi
+case "$command" in
+ prepare)
+ exec bash prepare.sh
+ ;;
+ package)
+ bash prepare.sh
+ mkdir -p dist
+ archive="raptor-artifact-${RAPTOR_COMMIT:0:12}.tar.gz"
+ package_dir=$(mktemp -d)
+ trap 'rm -rf -- "$package_dir"' EXIT
+ mkdir "$package_dir/raptor-artifact"
+ cp \
+ README.md artifact.sh prepare.sh Dockerfile .dockerignore pins.env \
+ requirements.lock models.sha256 build.sh run.py plot_ablation.py source.tar.gz source.sha256 \
+ "$package_dir/raptor-artifact/"
+ cp -R validation-overlay "$package_dir/raptor-artifact/"
+ cp validation-overlay.sha256 "$package_dir/raptor-artifact/"
+ tar -czf "dist/$archive" -C "$package_dir" raptor-artifact
+ (cd dist && sha256sum "$archive" > "$archive.sha256")
+ printf 'Attachment: %s/dist/%s\n' "$PWD" "$archive"
+ ;;
+ build)
+ [[ -f source.tar.gz ]] || bash prepare.sh
+ if [[ -f ../validation/raptor_validation/experiment_scheduler.py ]]; then
+ bash prepare.sh --overlay-only
+ fi
+ sha256sum -c source.sha256
+ docker build --build-arg "BUILD_JOBS=${BUILD_JOBS:-$(env -u OMP_NUM_THREADS -u OMP_THREAD_LIMIT nproc)}" -t "$image" .
+ ;;
+ reproduce|smoke|plots)
+ output=${OUTPUT_DIR:-"$PWD/results"}
+ mkdir -p "$output"
+ output=$(cd -- "$output" && pwd)
+ image_id=$(docker image inspect --format '{{.Id}}' "$image")
+ status=0
+ docker run --rm --init --network none \
+ --user "$(id -u):$(id -g)" \
+ --env "ARTIFACT_IMAGE_ID=$image_id" \
+ --env "JOBS=${JOBS:-$(env -u OMP_NUM_THREADS -u OMP_THREAD_LIMIT nproc)}" \
+ --env "THREADS_PER_JOB=${THREADS_PER_JOB:-$(env -u OMP_NUM_THREADS -u OMP_THREAD_LIMIT nproc)}" \
+ --mount "type=bind,src=$output,dst=/results" \
+ "$image" "$command" "$@" || status=$?
+ profile=full
+ if [[ "$command" = smoke || " $* " = *' --smoke '* ]]; then profile=smoke; fi
+ if [[ -f "$output/$profile/index.html" ]]; then
+ printf '\nOpen %s/%s/index.html\n' "$output" "$profile"
+ fi
+ exit "$status"
+ ;;
+ *)
+ echo 'Usage: bash artifact.sh {build|smoke|reproduce|plots|package}'
+ [[ "$command" = help ]]
+ ;;
+esac
diff --git a/artifact/build.sh b/artifact/build.sh
new file mode 100644
index 0000000..80c3535
--- /dev/null
+++ b/artifact/build.sh
@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+set -euo pipefail
+source /opt/artifact/pins.env
+export CMAKE_BUILD_PARALLEL_LEVEL=${1:-$(env -u OMP_NUM_THREADS -u OMP_THREAD_LIMIT nproc)}
+[[ "$CMAKE_BUILD_PARALLEL_LEVEL" =~ ^[1-9][0-9]*$ ]]
+cd /opt/raptor
+sha256sum -c /opt/artifact/models.sha256
+
+checkout() {
+ local url=$1 revision=$2 destination=$3
+ git init "$destination"
+ git -C "$destination" remote add origin "$url"
+ git -C "$destination" fetch --depth 1 origin "$revision"
+ git -C "$destination" checkout --detach FETCH_HEAD
+ test "$(git -C "$destination" rev-parse HEAD)" = "$revision"
+ git -C "$destination" submodule update --init --recursive --depth 1
+}
+
+checkout https://github.com/onnx/onnx-mlir.git "$ONNX_MLIR_COMMIT" onnx-mlir
+checkout https://github.com/llvm/llvm-project.git "$LLVM_COMMIT" onnx-mlir/llvm-project
+checkout https://github.com/HEAPLab/PIMCOMP-NN.git "$PIMCOMP_COMMIT" third_party/PIMCOMP-NN
+checkout https://github.com/HEAPLab/pimsim-nn.git "$PIMSIM_COMMIT" backend-simulators/pim/pimsim-nn
+checkout https://github.com/protocolbuffers/protobuf.git "$PROTOBUF_COMMIT" /opt/protobuf
+
+common=(-G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_POSITION_INDEPENDENT_CODE=ON
+ -DCMAKE_EXE_LINKER_FLAGS=-fuse-ld=mold
+ -DCMAKE_SHARED_LINKER_FLAGS=-fuse-ld=mold
+ -DCMAKE_MODULE_LINKER_FLAGS=-fuse-ld=mold)
+
+cmake -S /opt/protobuf -B /opt/protobuf/build "${common[@]}" \
+ -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_ABSL_PROVIDER=module \
+ -DABSL_PROPAGATE_CXX_STD=ON -DCMAKE_CXX_STANDARD=17
+cmake --build /opt/protobuf/build
+cmake --install /opt/protobuf/build
+ldconfig
+
+llvm=onnx-mlir/llvm-project
+cmake -S "$llvm/llvm" -B "$llvm/build_release" "${common[@]}" \
+ '-DLLVM_ENABLE_PROJECTS=mlir;clang' -DLLVM_ENABLE_RUNTIMES=openmp \
+ -DLLVM_TARGETS_TO_BUILD=host -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_ENABLE_RTTI=ON \
+ -DENABLE_LIBOMPTARGET=OFF -DLLVM_ENABLE_LIBEDIT=OFF \
+ -DLLVM_PARALLEL_LINK_JOBS="$CMAKE_BUILD_PARALLEL_LEVEL"
+cmake --build "$llvm/build_release"
+
+cmake -S . -B build_release "${common[@]}" \
+ -DONNX_MLIR_ACCELERATORS=PIM -DLLVM_ENABLE_ASSERTIONS=ON \
+ -DONNX_MLIR_ENABLE_JAVA=OFF -DONNX_MLIR_BUILD_TESTS=OFF \
+ -DMLIR_DIR="$PWD/$llvm/build_release/lib/cmake/mlir"
+cmake --build ./build_release
+
+cmake -S third_party/PIMCOMP-NN -B third_party/PIMCOMP-NN/build "${common[@]}"
+cmake --build third_party/PIMCOMP-NN/build --target PIMCOMP-NN
+sim=backend-simulators/pim/pimsim-nn
+cmake -S "$sim" -B "$sim/build" "${common[@]}"
+cmake --build "$sim/build"
+ctest --test-dir "$sim/build" --output-on-failure
+
+crate=backend-simulators/pim/pim-simulator
+cargo build --locked --release --no-default-features --manifest-path "$crate/Cargo.toml" \
+ --package pim-simulator --bin pim-simulator --jobs "$CMAKE_BUILD_PARALLEL_LEVEL"
+# The experiment driver uses cargo run. All dependencies must be available offline.
+cargo fetch --locked --manifest-path "$crate/Cargo.toml"
+mkdir -p /opt/artifact/provenance
+cp /opt/artifact/pins.env /opt/artifact/source.sha256 /opt/artifact/models.sha256 /opt/artifact/provenance/
+for dir in onnx-mlir "$llvm" third_party/PIMCOMP-NN "$sim" /opt/protobuf; do
+ printf '%s %s\n' "$dir" "$(git -C "$dir" rev-parse HEAD)"
+ git -C "$dir" submodule status --recursive
+done > /opt/artifact/provenance/revisions.txt
+dpkg-query -W > /opt/artifact/provenance/system-packages.txt
+.venv/bin/python -m pip freeze > /opt/artifact/provenance/python-packages.txt
+{ clang-19 --version; mold --version; cmake --version; rustc --version; } \
+ > /opt/artifact/provenance/toolchain.txt
+# Runtime uses an arbitrary host UID. Cargo and the runners need writable caches
+# and ONNX-MLIR writes temporary shims/configurations inside its source tree.
+chmod -R a+rwX /opt/raptor /opt/cargo /opt/rustup
diff --git a/artifact/models.sha256 b/artifact/models.sha256
new file mode 100644
index 0000000..9b4a30a
--- /dev/null
+++ b/artifact/models.sha256
@@ -0,0 +1,5 @@
+788088b908e233d924c7c26b997e89ee861290c7bc56783a306e8201d79aac8f validation/networks/pimcomp_models/resnet18/resnet18-v1-7.onnx
+c3231061d081bdd47884137b02134f85142752a39e87263c529cd14ed242b096 validation/networks/pimcomp_models/resnet34/resnet34-v1-7.onnx
+c960c195953423388733cd0e60712cf425ca1f8917e264f5cf98e028752c61cf validation/networks/pimcomp_models/googlenet/googlenet-12-no-lrn.onnx
+396cdea21e5e7d02c3f26f14d22ef20975171702493f5c5e79b8e0d896e541ef validation/networks/pimcomp_models/vgg8/vgg8-mnist-reconstructed.onnx
+5e7bd85dc60b252674f8d7e1ad24f2317d00dfb8c524826536698462de04620d validation/networks/pimcomp_models/yolo11n/yolo11n.onnx
diff --git a/artifact/pins.env b/artifact/pins.env
new file mode 100644
index 0000000..602db15
--- /dev/null
+++ b/artifact/pins.env
@@ -0,0 +1,8 @@
+# Raptor main at artifact preparation time. Do not replace this with a branch.
+RAPTOR_COMMIT=4a1fe710b2f0e4e9c0504f4fd9b4c4dd3df53b7d
+ONNX_MLIR_COMMIT=eb54c2afc46d00c6b196d1f275b6bfee17e12f69
+LLVM_COMMIT=0c2701fe7fa002e1befc5f86c268a7964f96d286
+PIMCOMP_COMMIT=7b6be6da4d33bf19f68d073eccac2a78e326a5af
+PIMSIM_COMMIT=17e37c15da5473362e53268dabdd07dff5fe8fbf
+PROTOBUF_COMMIT=6a6cd88c262ffdb1738167a47d5fcc7a3eb4edac
+RUST_VERSION=1.90.0
diff --git a/artifact/plot_ablation.py b/artifact/plot_ablation.py
new file mode 100644
index 0000000..9785486
--- /dev/null
+++ b/artifact/plot_ablation.py
@@ -0,0 +1,59 @@
+"""Plot planning and synchronization gains without hiding invalid programs."""
+from __future__ import annotations
+
+import argparse
+import csv
+import math
+from pathlib import Path
+
+import matplotlib
+matplotlib.use('Agg')
+import matplotlib.pyplot as plt
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument('--input', type=Path, required=True)
+ parser.add_argument('--output-dir', type=Path, required=True)
+ args = parser.parse_args()
+ with args.input.open(newline='') as stream:
+ rows = list(csv.DictReader(stream))
+ if not rows:
+ raise ValueError('The ablation CSV is empty')
+ panels = [('plan', 'latency', 'latency_gain_percent', 'Planning: latency reduction (%)'),
+ ('plan', 'throughput', 'throughput_gain_percent', 'Planning: throughput increase (%)'),
+ ('sync', 'throughput', 'throughput_gain_percent', 'Synchronization: throughput increase (%)')]
+ fig, axes = plt.subplots(3, 1, figsize=(11, 9), layout='constrained')
+ for ax, (comparison, mode, field, label) in zip(axes, panels):
+ selected = sorted((r for r in rows if r['comparison'] == comparison and r['mode'] == mode),
+ key=lambda r: (r['arch'], r['model'], int(r['raptor_pipeline'])))
+ for x, row in enumerate(selected):
+ try:
+ value = float(row[field].rstrip('%'))
+ except (ValueError, TypeError):
+ value = math.nan
+ valid = row['functional_validation'] == row['baseline_functional_validation'] == 'PASS'
+ if math.isfinite(value):
+ ax.bar(x, value, color='#B3D5CA' if valid else '#FF9494',
+ edgecolor='#222222', hatch='' if valid else '//')
+ ax.annotate(f'{value:.1f}' + ('' if valid else ' INVALID'),
+ (x, value), xytext=(0, 4 if value >= 0 else -4),
+ textcoords='offset points', ha='center',
+ va='bottom' if value >= 0 else 'top', fontsize=7)
+ else:
+ ax.text(x, 0, 'NA', ha='center', va='bottom', fontsize=8)
+ ax.set_xticks(range(len(selected)),
+ [f"{r['model']}\n{r['arch']} p{r['raptor_pipeline']}" for r in selected], fontsize=8)
+ ax.set_ylabel(label)
+ ax.axhline(0, color='#222222', linewidth=.7)
+ ax.set_xlim(-.6, max(.6, len(selected) - .4))
+ ax.margins(y=.3)
+ fig.suptitle('Raptor ablation (red hatching: functional validation failed)')
+ args.output_dir.mkdir(parents=True, exist_ok=True)
+ for extension in ('pdf', 'svg', 'png'):
+ fig.savefig(args.output_dir / f'ablation.{extension}', dpi=200)
+ plt.close(fig)
+
+
+if __name__ == '__main__':
+ main()
diff --git a/artifact/prepare.sh b/artifact/prepare.sh
new file mode 100644
index 0000000..5adbdb5
--- /dev/null
+++ b/artifact/prepare.sh
@@ -0,0 +1,39 @@
+#!/usr/bin/env bash
+set -euo pipefail
+cd -- "$(dirname -- "${BASH_SOURCE[0]}")"
+source pins.env
+repo=$(git -C .. rev-parse --show-toplevel)
+test "$(git -C "$repo" rev-parse "$RAPTOR_COMMIT^{commit}")" = "$RAPTOR_COMMIT"
+for item in "onnx-mlir:$ONNX_MLIR_COMMIT" \
+ "third_party/PIMCOMP-NN:$PIMCOMP_COMMIT" \
+ "backend-simulators/pim/pimsim-nn:$PIMSIM_COMMIT"; do
+ path=${item%:*}
+ expected=${item##*:}
+ actual=$(git -C "$repo" rev-parse "$RAPTOR_COMMIT:$path")
+ test "$actual" = "$expected"
+done
+if [[ "${1:-}" != --overlay-only ]]; then
+ # Include the complete compiler and evaluation code, plus only the suite models.
+ # The last two paths are the targets of the suite's relative model symlinks.
+ git -C "$repo" archive --format=tar "$RAPTOR_COMMIT" \
+ CMakeLists.txt README.md .gitmodules src test docs backend-simulators \
+ validation/README.md validation/validate.py validation/raptor_validation \
+ validation/tools validation/pimsim_configs validation/networks/pimcomp_models \
+ validation/networks/resnet18/depth_68/resnet18_depth_68.onnx \
+ validation/networks/yolo11n/depth_51/yolo11n_depth_51.onnx \
+ | gzip -n > source.tar.gz.tmp
+ mv source.tar.gz.tmp source.tar.gz
+ sha256sum source.tar.gz > source.sha256
+fi
+overlay=(validation/raptor_validation/experiment_scheduler.py
+ validation/tools/pim/pimcomp/compare/run_pimcomp_models.py
+ validation/tools/pim/pimcomp/compare/compare_raptor_pimcomp_model.py
+ validation/tools/pim/ablation/run_ablation.py
+ validation/tools/pim/pimcomp/compare/README.md
+ validation/tools/pim/ablation/README.md)
+for path in "${overlay[@]}"; do
+ mkdir -p "validation-overlay/$(dirname "$path")"
+ cp "$repo/$path" "validation-overlay/$path"
+done
+sha256sum "${overlay[@]/#/validation-overlay/}" > validation-overlay.sha256
+printf 'Prepared Raptor %s\n' "$RAPTOR_COMMIT"
diff --git a/artifact/requirements.lock b/artifact/requirements.lock
new file mode 100644
index 0000000..85e3f14
--- /dev/null
+++ b/artifact/requirements.lock
@@ -0,0 +1,23 @@
+# Evaluation and ONNX-MLIR build dependencies. The Graph Explorer is not required.
+cmake==3.31.6
+colorama==0.4.6
+contourpy==1.3.2
+cycler==0.12.1
+execnet==2.1.1
+fonttools==4.58.0
+iniconfig==2.1.0
+kiwisolver==1.4.8
+lit==15.0.7
+matplotlib==3.10.3
+numpy==2.2.6
+onnx==1.17.0
+packaging==25.0
+pillow==11.2.1
+pluggy==1.5.0
+protobuf==7.34.0
+pyparsing==3.2.3
+pytest==8.3.5
+pytest-xdist==3.6.1
+python-dateutil==2.9.0.post0
+PyYAML==6.0.2
+six==1.17.0
diff --git a/artifact/run.py b/artifact/run.py
new file mode 100644
index 0000000..cd73210
--- /dev/null
+++ b/artifact/run.py
@@ -0,0 +1,140 @@
+"""Container commands for the pinned Raptor paper artifact."""
+from __future__ import annotations
+
+import argparse
+import csv
+from datetime import datetime, timezone
+from html import escape
+import json
+import os
+from pathlib import Path
+import platform
+import shutil
+import subprocess
+import sys
+import time
+
+HERE = Path(__file__).resolve().parent
+REPO = Path('/opt/raptor')
+PYTHON = REPO / '.venv/bin/python'
+COMPARE = Path('validation/tools/pim/pimcomp/compare')
+ABLATION = Path('validation/tools/pim/ablation/run_ablation.py')
+
+
+def stage(name: str, command: list[str], root: Path) -> dict:
+ print(f'[{name}] {command}', flush=True)
+ start = time.monotonic()
+ with (root / f'{name}.log').open('wb') as log:
+ with subprocess.Popen(command, cwd=REPO, stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT) as process:
+ for line in process.stdout:
+ sys.stdout.buffer.write(line)
+ sys.stdout.buffer.flush()
+ log.write(line)
+ returncode = process.wait()
+ return {'command': command, 'exit_code': returncode,
+ 'seconds': round(time.monotonic() - start, 3)}
+
+
+def plots(root: Path, smoke: bool) -> dict:
+ commands = {'ablation-plots': [str(PYTHON), str(HERE / 'plot_ablation.py'),
+ '--input', str(root / 'ablation/results_ablation.csv'),
+ '--output-dir', str(root / 'plots/ablation')]}
+ if not smoke:
+ commands['comparison-plots'] = [str(PYTHON), str(COMPARE / 'generate_plots.py'),
+ '--input', str(root / 'comparison/results_comparison.csv'),
+ '--output-dir', str(root / 'plots/comparison')]
+ return {name: stage(name, command, root) for name, command in commands.items()}
+
+
+def show_results(root: Path) -> None:
+ page = ['',
+ '',
+ '
Raptor resultsRaptor results
',
+ 'Functional validation status is shown in the tables and plots.
']
+ for title, relative in (
+ ('Comparison', 'plots/comparison/png/performance_energy_compact.png'),
+ ('Pipeline throughput', 'plots/comparison/png/raptor_pipeline_throughput.png'),
+ ('Ablation', 'plots/ablation/ablation.png'),
+ ):
+ if (root / relative).is_file():
+ page.append(f'{title}
')
+ for title, relative in (('Comparison', 'comparison/results_comparison.csv'),
+ ('Ablation', 'ablation/results_ablation.csv')):
+ path = root / relative
+ if path.is_file():
+ page.append(f'{title} data
Download CSV
')
+ with path.open(newline='') as stream:
+ for index, row in enumerate(csv.reader(stream)):
+ tag = 'th' if index == 0 else 'td'
+ page.append('' + ''.join(f'<{tag}>{escape(value)}{tag}>' for value in row) + '
')
+ page.append('
')
+ page.append('Files
')
+ for path in sorted([*root.glob('*.log'), *root.glob('plots/*')]):
+ relative = path.relative_to(root).as_posix()
+ page.append(f'- {escape(relative)}
')
+ page.append('
')
+ (root / 'index.html').write_text('\n'.join(page) + '\n')
+
+
+def main() -> int:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument('command', choices=('reproduce', 'smoke', 'plots'))
+ parser.add_argument('--smoke', action='store_true', help='Plot the smoke results.')
+ parser.add_argument('--jobs', type=int, default=int(os.environ.get('JOBS', len(os.sched_getaffinity(0)))))
+ parser.add_argument('--threads-per-job', type=int,
+ default=int(os.environ.get('THREADS_PER_JOB', len(os.sched_getaffinity(0)))))
+ args = parser.parse_args()
+ smoke = args.command == 'smoke' or args.smoke
+ root = Path('/results') / ('smoke' if smoke else 'full')
+ os.environ['CARGO_NET_OFFLINE'] = 'true'
+ if args.command == 'plots':
+ manifest_path = root / 'run.json'
+ manifest = json.loads(manifest_path.read_text())
+ records = plots(root, smoke)
+ manifest['stages'].update(records)
+ manifest_path.write_text(json.dumps(manifest, indent=2) + '\n')
+ show_results(root)
+ return int(any(record['exit_code'] for record in records.values()))
+ if root.exists():
+ parser.error(f'{root} exists. Select a fresh OUTPUT_DIR to prevent stale result reuse.')
+ jobs = args.jobs
+ if jobs < 1 or args.threads_per_job < 1:
+ parser.error('jobs and threads-per-job must be positive')
+ root.mkdir(parents=True)
+ shutil.copytree(HERE / 'provenance', root / 'provenance')
+ # Copy the artifact scripts so each result identifies its orchestration code.
+ shutil.copytree(HERE, root / 'provenance/artifact',
+ ignore=shutil.ignore_patterns('provenance', '__pycache__'))
+ manifest = {'started_utc': datetime.now(timezone.utc).isoformat(),
+ 'profile': 'smoke' if smoke else 'full', 'image_id': os.environ.get('ARTIFACT_IMAGE_ID'),
+ 'platform': platform.platform(), 'machine': platform.machine(),
+ 'cpu_count': os.cpu_count(), 'jobs': jobs, 'threads_per_job': args.threads_per_job,
+ 'batch_size': 64, 'fast': False, 'stages': {}}
+ common = ['--jobs', str(jobs), '--threads-per-job', str(args.threads_per_job),
+ '--batch-size', '64', '--no-fast']
+ subset = ['--models', 'vgg8', '--archs', 'arch-a', '--mode', 'latency', '--pipeline', '1'] if smoke else []
+ commands = {
+ 'experiments': [str(PYTHON), str(COMPARE / 'run_pimcomp_models.py'),
+ '--out-dir', str(root / 'comparison'),
+ '--with-ablation', '--ablation-out-dir', str(root / 'ablation'),
+ '--common-dir', str(root / 'references'), *common, *subset],
+ }
+ manifest_path = root / 'run.json'
+ manifest_path.write_text(json.dumps(manifest, indent=2) + '\n')
+ for name, command in commands.items():
+ manifest['stages'][name] = stage(name, command, root)
+ manifest_path.write_text(json.dumps(manifest, indent=2) + '\n')
+ manifest['stages'].update(plots(root, smoke))
+ manifest['finished_utc'] = datetime.now(timezone.utc).isoformat()
+ manifest_path.write_text(json.dumps(manifest, indent=2) + '\n')
+ show_results(root)
+ return int(any(record['exit_code'] for record in manifest['stages'].values()))
+
+
+if __name__ == '__main__':
+ raise SystemExit(main())
diff --git a/validation/raptor_validation/experiment_scheduler.py b/validation/raptor_validation/experiment_scheduler.py
new file mode 100644
index 0000000..6bfc411
--- /dev/null
+++ b/validation/raptor_validation/experiment_scheduler.py
@@ -0,0 +1,153 @@
+"""Run experiment subprocesses as their dependencies complete."""
+from __future__ import annotations
+
+import argparse
+from collections import deque
+from dataclasses import dataclass
+import json
+import os
+from pathlib import Path
+import signal
+import subprocess
+import time
+from typing import Callable
+
+
+def available_processors() -> int:
+ if hasattr(os, "sched_getaffinity"):
+ return len(os.sched_getaffinity(0))
+ return os.cpu_count() or 1
+
+
+def positive_integer(value: str) -> int:
+ number = int(value)
+ if number < 1:
+ raise argparse.ArgumentTypeError("must be positive")
+ return number
+
+
+def thread_environment(threads: int) -> dict[str, str]:
+ return {**os.environ, "OMP_NUM_THREADS": str(threads),
+ "OMP_THREAD_LIMIT": str(threads), "OMP_DYNAMIC": "FALSE",
+ "OMP_WAIT_POLICY": "PASSIVE", "OPENBLAS_NUM_THREADS": str(threads),
+ "MKL_NUM_THREADS": str(threads), "RAYON_NUM_THREADS": str(threads),
+ "CMAKE_BUILD_PARALLEL_LEVEL": str(threads), "CARGO_BUILD_JOBS": str(threads)}
+
+
+@dataclass(frozen=True)
+class ExperimentTask:
+ key: str
+ command: Callable[[], list[str]]
+ dependencies: tuple[str, ...]
+ log_path: Path
+
+
+@dataclass(frozen=True)
+class TaskResult:
+ returncode: int
+ error: str = ""
+
+
+def stop_processes(processes: list[subprocess.Popen]) -> None:
+ for process in processes:
+ try:
+ os.killpg(process.pid, signal.SIGTERM)
+ except ProcessLookupError:
+ pass
+ deadline = time.monotonic() + 5
+ for process in processes:
+ try:
+ process.wait(timeout=max(0, deadline - time.monotonic()))
+ except subprocess.TimeoutExpired:
+ try:
+ os.killpg(process.pid, signal.SIGKILL)
+ except ProcessLookupError:
+ pass
+ process.wait()
+
+
+def run_tasks(tasks: list[ExperimentTask], jobs: int, threads: int,
+ cwd: Path) -> dict[str, TaskResult]:
+ if jobs < 1 or threads < 1:
+ raise ValueError("jobs and threads-per-job must be positive")
+ by_key = {task.key: task for task in tasks}
+ if len(by_key) != len(tasks):
+ raise ValueError("duplicate experiment task")
+ waiting = {task.key: set(task.dependencies) for task in tasks}
+ children = {key: [] for key in by_key}
+ for task in tasks:
+ for dependency in task.dependencies:
+ if dependency not in by_key:
+ raise ValueError(f"unknown dependency: {dependency}")
+ children[dependency].append(task.key)
+ ready = deque(key for key in by_key if not waiting[key])
+ degrees = {key: len(dependencies) for key, dependencies in waiting.items()}
+ pending = deque(ready)
+ visited = 0
+ while pending:
+ visited += 1
+ for child in children[pending.popleft()]:
+ degrees[child] -= 1
+ if not degrees[child]:
+ pending.append(child)
+ if visited != len(tasks):
+ raise ValueError("experiment dependency cycle")
+ active = {}
+ results = {}
+ records = {}
+ environment = thread_environment(threads)
+
+ def complete(key: str, result: TaskResult) -> None:
+ results[key] = result
+ task = by_key[key]
+ record = records.get(key, {"dependencies": task.dependencies, "threads_per_job": threads})
+ record.update(returncode=result.returncode, error=result.error)
+ task.log_path.parent.mkdir(parents=True, exist_ok=True)
+ task.log_path.with_suffix(".execution.json").write_text(json.dumps(record, indent=2) + "\n")
+ print(f"[{'Failed' if result.returncode else 'Completed'}] {key}"
+ f"{': ' + result.error if result.error else ''}", flush=True)
+ for child in children[key]:
+ waiting[child].remove(key)
+ if not waiting[child]:
+ ready.append(child)
+
+ try:
+ while ready or active:
+ while ready and len(active) < jobs:
+ key = ready.popleft()
+ task = by_key[key]
+ failed = [dep for dep in task.dependencies if results[dep].returncode]
+ if failed:
+ complete(key, TaskResult(1, "prerequisite failed: " + ", ".join(failed)))
+ continue
+ log = None
+ try:
+ command = task.command()
+ task.log_path.parent.mkdir(parents=True, exist_ok=True)
+ log = task.log_path.open("wb")
+ metadata = {"command": command, "threads_per_job": threads,
+ "dependencies": task.dependencies}
+ records[key] = metadata
+ task.log_path.with_suffix(".execution.json").write_text(json.dumps(metadata, indent=2) + "\n")
+ process = subprocess.Popen(command, cwd=cwd, env=environment,
+ stdout=log, stderr=subprocess.STDOUT, start_new_session=True)
+ except (OSError, ValueError, RuntimeError) as exc:
+ if log is not None:
+ log.close()
+ complete(key, TaskResult(1, str(exc)))
+ continue
+ active[key] = process, log
+ print(f"[Started] {key}; log: {task.log_path}", flush=True)
+ completed = [(key, process.returncode) for key, (process, _) in active.items()
+ if process.poll() is not None]
+ for key, returncode in completed:
+ _, log = active.pop(key)
+ log.close()
+ complete(key, TaskResult(returncode))
+ if active and not completed:
+ time.sleep(.05)
+ return results
+ finally:
+ stop_processes([process for process, _ in active.values()])
+ for _, log in active.values():
+ log.close()
diff --git a/validation/tests/test_experiment_scheduler.py b/validation/tests/test_experiment_scheduler.py
new file mode 100644
index 0000000..d629536
--- /dev/null
+++ b/validation/tests/test_experiment_scheduler.py
@@ -0,0 +1,187 @@
+from __future__ import annotations
+
+import argparse
+import json
+from pathlib import Path
+import sys
+
+import pytest
+
+sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tools/pim/pimcomp/compare"))
+from raptor_validation import experiment_scheduler as scheduler
+import run_pimcomp_models as suite
+import compare_raptor_pimcomp_model as comparator
+sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tools/pim/ablation"))
+import run_ablation as ablation
+
+
+def task(root, key, code, dependencies=()):
+ command = [sys.executable, "-c", code]
+ return scheduler.ExperimentTask(key, lambda: command, dependencies, root / f"{key}.log")
+
+
+def test_dependency_starts_before_unrelated_job_finishes(tmp_path):
+ # The slow job cannot finish until the fast job's dependent has run.
+ marker = tmp_path / "released"
+ slow = ("import pathlib,time; deadline=time.monotonic()+5; "
+ f"p=pathlib.Path({str(marker)!r}); "
+ "exec('while not p.exists() and time.monotonic() 1:
- output.parent.mkdir(parents=True, exist_ok=True)
- output.write_text('{"new": true}')
- return [], offset + len(jobs)
+ def run_jobs(tasks, jobs, threads, cwd):
+ assert jobs == 1
+ results = {}
+ for task in tasks:
+ try:
+ command = task.command()
+ except ValueError as exc:
+ errors.append(str(exc))
+ results[task.key] = suite.TaskResult(1, str(exc))
+ continue
+ if "--prepare-common" not in command:
+ commands.append(command)
+ output = Path(command[command.index("--out-dir") + 1]) / "pimcomp/comparison_report.json"
+ if fresh or only == "raptor":
+ output.parent.mkdir(parents=True, exist_ok=True)
+ output.write_text('{"new": true}')
+ results[task.key] = suite.TaskResult(0)
+ return results
summary = tmp_path / "results.csv"
summary.write_text("summary\n")
monkeypatch.setattr(suite, "validate_pimcomp_source", lambda: None)
- monkeypatch.setattr(suite, "run", lambda *a, **k: 0)
- monkeypatch.setattr(suite, "run_comparison_jobs", run_jobs)
+ monkeypatch.setattr(suite, "run_tasks", run_jobs)
monkeypatch.setattr(suite, "write_results_csv", lambda *a: summary)
monkeypatch.setattr(suite, "comparison_passed", lambda report: True)
monkeypatch.setattr(sys, "argv", [
@@ -548,15 +557,19 @@ def test_suite_reruns_anchor_and_shares_only_new_reports(tmp_path, monkeypatch,
"--archs", "arch-a", "--mode", "throughput", "--jobs", "1",
*(["--only", only] if only else []),
])
- assert suite.main() == 0
+ assert suite.main() == (1 if only != "raptor" and not fresh else 0)
if only == "raptor":
- assert len(batches[0]) == 3
- assert all("--reuse-pimcomp-report" in command for command in batches[0])
+ assert len(commands) == 3
+ assert all("--reuse-pimcomp-report" in command for command in commands)
else:
- assert len(batches[0]) == 1
- assert "--reuse-pimcomp-report" not in batches[0][0]
- assert all(("--reuse-pimcomp-report" in command) == fresh for command in batches[1])
- assert ("--reuse-raptor-report" in batches[0][0]) == (only == "pimcomp")
+ assert "--reuse-pimcomp-report" not in commands[0]
+ assert ("--reuse-raptor-report" in commands[0]) == (only == "pimcomp")
+ if fresh:
+ assert len(commands) == 3
+ assert all("--reuse-pimcomp-report" in command for command in commands[1:])
+ else:
+ assert len(commands) == 1 # Missing reports must not cause duplicate Pimcomp work.
+ assert len(errors) == 2
def test_missing_pimcomp_report_is_not_reused(tmp_path):
@@ -658,7 +671,7 @@ def test_ablation_cli_selects_canonical_or_custom_output(tmp_path, monkeypatch,
canonical = tmp_path / "scripts/results_ablation.csv"
monkeypatch.setattr(ablation, "DEFAULT_OUT_DIR", artifact_root)
monkeypatch.setattr(ablation, "DEFAULT_RESULTS_PATH", canonical)
- monkeypatch.setattr(ablation.subprocess, "Popen", lambda *a, **k: argparse.Namespace(wait=lambda: 0))
+ monkeypatch.setattr(suite, "run_plans", lambda plans: 0)
calls = []
monkeypatch.setattr(ablation, "aggregate_results", lambda root, *a, **k: (calls.append((root, k["output"])) or None, []))
command = ["--variant", "baseline", "--jobs", "1"]
@@ -863,7 +876,13 @@ def test_ablation_runner_distinguishes_invalid_outputs_from_failed_execution(tmp
csv = tmp_path / "results.csv"
csv.write_text("summary\n")
monkeypatch.setattr(suite, "run", lambda *a, **k: 0)
- monkeypatch.setattr(suite, "run_comparison_jobs", lambda *a: (["arch-a/vgg8/throughput/pipeline4"] if process_failed else [], 1))
+ def complete_tasks(tasks, *args):
+ previous = output.stat().st_mtime_ns
+ output.write_text(json.dumps(report))
+ os.utime(output, ns=(previous + 1000000, previous + 1000000))
+ return {task.key: suite.TaskResult(int(process_failed)) for task in tasks}
+
+ monkeypatch.setattr(suite, "run_tasks", complete_tasks)
monkeypatch.setattr(suite, "write_results_csv", lambda *a: csv)
monkeypatch.setattr(sys, "argv", ["run_pimcomp_models.py", "--out-dir", str(tmp_path),
"--models", "vgg8", "--archs", "arch-a", "--mode", "throughput", "--pipeline", "4",
diff --git a/validation/tools/pim/ablation/README.md b/validation/tools/pim/ablation/README.md
index 5407482..1a27c0c 100644
--- a/validation/tools/pim/ablation/README.md
+++ b/validation/tools/pim/ablation/README.md
@@ -9,6 +9,9 @@ Run the synchronization/Spatial-planning ablation from the repository root:
Defaults: `vgg8 resnet18 resnet34 googlenet`, `arch-a arch-b`, latency
(pipeline 1), and throughput (pipeline 4). Select `yolo11n` or `arch-c`
explicitly when needed. Only Raptor runs; Pimcomp validation is `SKIPPED`.
+Ready models, architectures, modes, and variants run concurrently in one dependency queue.
+References are prepared once per model. `--jobs` and `--threads-per-job` both default
+to the available logical processor count; override them to limit CPU contention.
| Variant | Raptor options |
|---|---|
@@ -34,7 +37,7 @@ suite pipelines for that mode. Pipeline-only selection uses its matching mode.
`--pipeline auto` requests automatic throughput depth selection; positive numeric
depths retain their exact-stage meaning.
`--only` is rejected because
-the wrapper always runs Raptor only. Ctrl+C stops the active variant's workers.
+the wrapper always runs Raptor only. Ctrl+C stops all active experiment workers.
Artifacts use `//artifacts//[/pipelineN]/ablation//`;
reference inputs, outputs, and runners are shared under `artifacts/common/`.
diff --git a/validation/tools/pim/ablation/run_ablation.py b/validation/tools/pim/ablation/run_ablation.py
index 15cff0b..d58d138 100755
--- a/validation/tools/pim/ablation/run_ablation.py
+++ b/validation/tools/pim/ablation/run_ablation.py
@@ -6,10 +6,7 @@ from __future__ import annotations
import argparse
import csv
import math
-import os
import shlex
-import signal
-import subprocess
import sys
from dataclasses import dataclass
from pathlib import Path
@@ -26,6 +23,7 @@ from raptor_validation.pimcomp_models import (
)
from raptor_validation.artifacts import remove_lock_files
from raptor_validation.raptor import parse_pipeline_depth, pipeline_sort_key
+from raptor_validation.experiment_scheduler import available_processors, positive_integer
@dataclass(frozen=True)
@@ -79,9 +77,14 @@ def parse_args(argv: list[str] | None = None) -> tuple[argparse.Namespace, list[
help="Run only this variant; baseline is added when needed. Repeat as needed.",
)
parser.add_argument("--dry-run", action="store_true", help="Print runner commands without modifying files.")
+ parser.add_argument("--jobs", "-j", type=positive_integer, default=available_processors(),
+ help="Maximum concurrent cases (default: available logical processors).")
+ parser.add_argument("--threads-per-job", type=positive_integer, default=available_processors(),
+ help="Threads per case (default: available logical processors).")
args, forwarded = parser.parse_known_args(argv)
if any(option == "--only" or option.startswith("--only=") for option in forwarded):
parser.error("--only is not supported; the ablation wrapper always runs Raptor only")
+ forwarded.extend(("--jobs", str(args.jobs), "--threads-per-job", str(args.threads_per_job)))
return args, forwarded
@@ -156,23 +159,6 @@ def runner_command(
return command
-def terminate_process_group(process: subprocess.Popen[bytes]) -> None:
- if process.poll() is not None:
- return
- try:
- os.killpg(process.pid, signal.SIGTERM)
- except ProcessLookupError:
- return
- try:
- process.wait(timeout=5)
- except subprocess.TimeoutExpired:
- try:
- os.killpg(process.pid, signal.SIGKILL)
- except ProcessLookupError:
- return
- process.wait()
-
-
def percentage_difference(
value: str | None, reference: str | None, *, lower_is_better: bool = False
) -> str:
@@ -313,33 +299,27 @@ def main(argv: list[str] | None = None) -> int:
)
return 0
- out_dir.mkdir(parents=True, exist_ok=True)
- failed = []
- current_process = None
+ sys.path.insert(0, str(RUNNER.parent))
+ import run_pimcomp_models as suite
+ plans = []
try:
for variant in variants:
- for case_label, case_forwarded in case_sets:
+ for _, case_forwarded in case_sets:
if variant.name == "no-sync" and case_key(case_forwarded)[0] != "throughput":
continue
- command = runner_command(
- out_dir, variant, args.models, case_forwarded, out_dir, False
- )
- print(f"[{variant.name}/{case_label}] {shlex.join(command)}")
- current_process = subprocess.Popen(
- command,
- cwd=REPO,
- start_new_session=True,
- )
- returncode = current_process.wait()
- current_process = None
- if returncode:
- failed.append(f"{variant.name}/{case_label}: runner exited with {returncode}")
+ command = runner_command(out_dir, variant, args.models, case_forwarded, out_dir, False)
+ selected = suite.parse_args(command[2:])
+ if selected.clean:
+ print(f"Removed {suite.clean_artifacts(out_dir, args.models, out_dir)} artifact path(s).")
+ return 0
+ plans.append(suite.make_plan(selected))
+ failed = ["experiment jobs failed"] if suite.run_plans(plans) else []
except KeyboardInterrupt:
- if current_process is not None:
- terminate_process_group(current_process)
- remove_lock_files(out_dir)
- print("Interrupted; terminated the active ablation job.", file=sys.stderr)
+ print("Interrupted; terminated ablation workers.", file=sys.stderr)
return 130
+ except ValueError as exc:
+ print(str(exc), file=sys.stderr)
+ return 2
remove_lock_files(out_dir)
selected_cases = frozenset(case_key(arguments) for _, arguments in case_sets)
diff --git a/validation/tools/pim/pimcomp/compare/README.md b/validation/tools/pim/pimcomp/compare/README.md
index 34a1e8b..8c6aef8 100644
--- a/validation/tools/pim/pimcomp/compare/README.md
+++ b/validation/tools/pim/pimcomp/compare/README.md
@@ -11,7 +11,12 @@ built release Raptor, Pimcomp, and `pimsim-nn`.
```
Defaults: all five models, Arch-A/B, latency pipeline 1, throughput pipelines
-2/4/8, four workers, and a functional throughput batch of 64.
+2/4/8 and a functional throughput batch of 64.
+Both `--jobs` and `--threads-per-job` default to the available logical processor count.
+Their limits multiply: 40 jobs with 40 threads can create 1,600 threads.
+Use `--jobs 8 --threads-per-job 4` to reduce contention.
+The thread limit controls OpenMP, Raptor's compilation thread pool (`-j`), BLAS, and Rayon.
+Reference batches use that many workers, with one thread per reference process.
Use `--help` for the complete option reference. Numeric depths greater than one
require exactly that many active compute stages. `--pipeline auto` requests
throughput depth selection. Reports include `raptor_selected_pipeline`; the
@@ -26,6 +31,8 @@ requested depth remains in the existing pipeline field.
| `--only {raptor,pimcomp}` | Rerun one compiler and reuse the other's existing report. |
| `--raptor-only` | Run Raptor without invoking Pimcomp. |
| `--batch-size N`, `--jobs N` | Set functional throughput batch size and worker count. |
+| `--threads-per-job N` | Set the thread limit inside each job. |
+| `--with-ablation`, `--ablation-out-dir PATH` | Schedule both suites together; optionally select a separate ablation root. |
| `--pimsim-time-ms N` | Throughput convergence deadline; default: 1000 ms. |
| `--no-fast` | Use full-duration throughput simulation for authoritative experiments. |
| `--timeout-seconds N` | Per-stage timeout; default: 0 (unlimited). |
@@ -45,11 +52,19 @@ Artifacts use
under `artifacts/common/`
and `comparison_report.{md,json}` under each comparison's `pimcomp/`.
YOLO and GoogLeNet use `artifacts/common-full-model/` to avoid reusing historical Softmax-free references.
+References are prepared once per model. Each case starts when its own prerequisites finish.
Pimcomp runs once per model/architecture/mode; later pipelines reuse its newly
written report, including failures without simulation artifacts. Missing or
-unchanged anchor reports trigger fresh attempts. The runner continues after failures and exits nonzero if a
+unchanged prerequisite reports skip dependent cases without repeating Pimcomp work.
+Skipped cases retain unavailable metrics rather than old measurements.
+The runner continues after failures and exits nonzero if a
compiler, simulation, or functional comparison fails.
+Combined runs share references with ablation and reuse matching Raptor baseline reports.
+`ablation/baseline/source_reports.json` records those report locations.
+Only the parent process writes aggregate CSVs. Per-case logs and execution settings remain beside each case.
+Thread limits are recorded in reports and checked during report reuse.
+
If fast-mode convergence fails, the reports and CSV retain valid full-duration
fallback throughput and output interval values. This is not a converged steady-state result.
Per-case JSON records `throughput_measurement: full_duration_fallback` and
diff --git a/validation/tools/pim/pimcomp/compare/compare_raptor_pimcomp_model.py b/validation/tools/pim/pimcomp/compare/compare_raptor_pimcomp_model.py
index c3086a3..b54327c 100755
--- a/validation/tools/pim/pimcomp/compare/compare_raptor_pimcomp_model.py
+++ b/validation/tools/pim/pimcomp/compare/compare_raptor_pimcomp_model.py
@@ -46,6 +46,7 @@ from raptor_validation.onnx_utils import (
)
from raptor_validation.raptor import compile_with_raptor, read_selected_pipeline_depth
from raptor_validation.output_comparison import VALIDATION_POLICY, compare_output_arrays
+from raptor_validation.experiment_scheduler import available_processors, positive_integer, thread_environment
from raptor_validation.pimsim_nn import (
export_raptor_pimsim_artifact,
parse_pimsim_nn_metrics,
@@ -156,6 +157,7 @@ def run_logged(
steps: list[StepRecord],
stage: str | None = None,
print_header: bool = True,
+ env: dict[str, str] | None = None,
) -> str:
if print_header:
print_step(name, cmd, cwd, stage)
@@ -172,6 +174,7 @@ def run_logged(
stderr=subprocess.STDOUT,
text=True,
timeout=None if timeout_sec == 0 else timeout_sec,
+ env=env,
)
except subprocess.TimeoutExpired as exc:
duration = time.perf_counter() - start
@@ -364,7 +367,8 @@ def compile_reference(
run_logged(
"Compile reference ONNX IR",
[str(args.raptor_path), str(model_path), "-o", str(onnx_ir_base), "--EmitONNXIR",
- "--mlir-elide-elementsattrs-if-larger=16", "--enable-conv-opt-pass=false"],
+ "--mlir-elide-elementsattrs-if-larger=16", "--enable-conv-opt-pass=false",
+ f"-j={args.threads_per_job}"],
cwd=REPO,
timeout_sec=args.timeout_seconds,
steps=steps,
@@ -372,7 +376,7 @@ def compile_reference(
)
run_logged(
"Compile reference native",
- [str(args.raptor_path), "-O3", str(model_path), "-o", str(runner_base)],
+ [str(args.raptor_path), "-O3", str(model_path), "-o", str(runner_base), f"-j={args.threads_per_job}"],
cwd=REPO,
timeout_sec=args.timeout_seconds,
steps=steps,
@@ -400,7 +404,7 @@ def compile_reference(
)
run_logged(
"Build runner",
- ["cmake", "--build", ".", "-j"],
+ ["cmake", "--build", ".", "--parallel", str(args.threads_per_job)],
cwd=build_dir,
timeout_sec=args.timeout_seconds,
steps=steps,
@@ -446,6 +450,7 @@ def generate_reference_outputs(
timeout_sec=args.timeout_seconds,
steps=steps,
print_header=print_header,
+ env=thread_environment(1),
)
return reference_dir
@@ -576,6 +581,7 @@ def compile_raptor_target(
f"--core-count={hardware['core_count']}",
f"--pim-target-config={args.pimcomp_config}",
"--pim-emit-json",
+ f"-j={args.threads_per_job}",
*args.raptor_extra_arg,
]
print_step("Compile Raptor", cmd, REPO, "Compile Pim")
@@ -584,6 +590,7 @@ def compile_raptor_target(
raptor_extra_args = [
f"--pim-target-config={args.pimcomp_config}",
"--pim-emit-json",
+ f"-j={args.threads_per_job}",
*args.raptor_extra_arg,
]
try:
@@ -1202,6 +1209,7 @@ def experiment_context(args, model, functional_model, hardware, side):
"oracle": args.pimsim_nn_build_dir / "ChipTest",
"comparison": Path(__file__),
"validation": VALIDATION_DIR / "raptor_validation/output_comparison.py",
+ "thread_policy": VALIDATION_DIR / "raptor_validation/experiment_scheduler.py",
"functional_simulator": args.pim_simulator_dir / "target/release/pim-simulator"}
if side == "raptor":
files["compiler"] = args.raptor_path
@@ -1216,6 +1224,7 @@ def experiment_context(args, model, functional_model, hardware, side):
settings.update({name: getattr(args, name) for name in (
("raptor_extra_arg",) if side == "raptor" else
("pimcomp_pipeline", "pimcomp_replication", "pimcomp_model_name"))})
+ settings["threads_per_job"] = getattr(args, "threads_per_job", available_processors())
return {"version": 2, "validation_policy": VALIDATION_POLICY,
"files": fingerprint_files(files), "hardware": hardware, "settings": settings}
@@ -1569,9 +1578,9 @@ def main():
parser.add_argument(
"-j",
"--jobs",
- type=int,
- default=4,
- help="Maximum parallel native reference runner processes (default: 4).",
+ type=positive_integer,
+ default=available_processors(),
+ help="Maximum parallel native reference runner processes (default: logical processors).",
)
parser.add_argument("--pimcomp-pipeline", choices=["element", "batch"])
parser.add_argument("--pimcomp-model-name", help="Use a Pimcomp built-in model name such as vgg16.")
@@ -1613,7 +1622,10 @@ def main():
action="store_true",
help="Return a non-zero status if a stage or semantic validation fails.",
)
+ parser.add_argument("--threads-per-job", type=positive_integer, default=available_processors(),
+ help="Threads for OpenMP, MLIR compilation, and simulation (default: logical processors).")
args = parser.parse_args()
+ os.environ.update(thread_environment(args.threads_per_job))
if args.raptor_only and any(
option is not None
for option in (
@@ -2152,6 +2164,7 @@ def main():
"hardware": hardware,
"pimsim_mode": args.pimsim_mode,
"pimsim_time_ms": args.pimsim_time_ms,
+ "threads_per_job": args.threads_per_job,
"pimcomp_pipeline": args.pimcomp_pipeline,
"pimcomp_replication": args.pimcomp_replication,
"pimcomp_model_source": "not_run" if args.raptor_only else "original_onnx",
diff --git a/validation/tools/pim/pimcomp/compare/run_pimcomp_models.py b/validation/tools/pim/pimcomp/compare/run_pimcomp_models.py
index 00d60b0..9af71a6 100755
--- a/validation/tools/pim/pimcomp/compare/run_pimcomp_models.py
+++ b/validation/tools/pim/pimcomp/compare/run_pimcomp_models.py
@@ -5,16 +5,12 @@ import argparse
import csv
import json
import math
-import os
import shlex
import shutil
import subprocess
import sys
-from concurrent.futures import ProcessPoolExecutor, as_completed
-from contextlib import nullcontext, redirect_stderr, redirect_stdout
from dataclasses import dataclass
from pathlib import Path
-from tempfile import TemporaryDirectory
from colorama import Fore, Style
@@ -31,7 +27,9 @@ from raptor_validation.artifacts import artifacts_dir, remove_lock_files
from raptor_validation.pimsim_nn import parse_pimsim_nn_metrics
from raptor_validation.output_comparison import VALIDATION_POLICY
from raptor_validation.raptor import parse_pipeline_depth, pipeline_sort_key
-from raptor_validation.validate_one import STAGE_COLORS
+from raptor_validation.experiment_scheduler import (
+ ExperimentTask, TaskResult, available_processors, positive_integer, run_tasks,
+)
PIMCOMP_SOURCE = REPO / "third_party/PIMCOMP-NN"
PIMCOMP_CONFIGS = REPO / "validation/pimsim_configs/pimcomp"
@@ -130,6 +128,8 @@ def write_results_csv(
models: list[str],
comparisons: tuple[tuple[str, int, str], ...] = COMPARISONS,
ablation_variant: str | None = None,
+ report_paths: dict[Path, Path | None] | None = None,
+ raptor_only: bool = False,
) -> Path:
output = DEFAULT_RESULTS_PATH if root is None else root / RESULTS_FILENAME
if ablation_variant is not None:
@@ -179,6 +179,9 @@ def write_results_csv(
report_path = result_dir(
root, name, arch, comparison_mode, pipeline, ablation_variant
) / "pimcomp/comparison_report.json"
+ if report_paths is not None:
+ directory = result_dir(root, name, arch, comparison_mode, pipeline, ablation_variant)
+ report_path = report_paths.get(directory, report_path)
row = {
"model": name,
"arch": arch,
@@ -198,12 +201,12 @@ def write_results_csv(
"better_compiler": "NA",
"speedup": "NA",
}
- if report_path.exists():
+ if report_path is not None and report_path.exists():
report = json.loads(report_path.read_text(encoding="utf-8"))
if report.get("validation_policy") != VALIDATION_POLICY:
raise ValueError(f"Report uses a different validation policy: {report_path}. Regenerate this comparison.")
raptor_values = performance_values(report.get("raptor_performance") or {})
- pimcomp_values = performance_values(report.get("pimcomp_performance") or {})
+ pimcomp_values = performance_values({} if raptor_only else report.get("pimcomp_performance") or {})
raptor_metric = raptor_values["throughput"] if comparison_mode == "throughput" else raptor_values["latency"]
pimcomp_metric = pimcomp_values["throughput"] if comparison_mode == "throughput" else pimcomp_values["latency"]
row.update(
@@ -211,7 +214,7 @@ def write_results_csv(
report.get("raptor_validation")
),
pimcomp_functional_validation=functional_validation_status(
- report.get("pimcomp_validation")
+ {"status": "skipped"} if raptor_only else report.get("pimcomp_validation")
),
raptor_throughput_samples_s=format_value(raptor_values["throughput"]),
pimcomp_throughput_samples_s=format_value(pimcomp_values["throughput"]),
@@ -322,17 +325,6 @@ def format_value(value: float | None) -> str:
return f"{value:.6g}"
-def print_stage(title: str, color: str) -> None:
- print(Style.BRIGHT + color + f"[{title}]" + Style.RESET_ALL, flush=True)
-
-
-def print_completed(label: str, output: str = "") -> None:
- print(Style.BRIGHT + Fore.CYAN + f"[Completed {label}]" + Style.RESET_ALL)
- if output:
- print(output, end="" if output.endswith("\n") else "\n")
- print("=" * 72, flush=True)
-
-
def run(command: list[str], *, dry_run: bool, check: bool = True) -> int:
print(f" cwd: {REPO}", flush=True)
print(f" $ {shlex.join(command)}", flush=True)
@@ -341,29 +333,6 @@ def run(command: list[str], *, dry_run: bool, check: bool = True) -> int:
return subprocess.run(command, cwd=REPO, check=check).returncode
-def run_comparison_job(job: tuple[str, list[str], Path | None]) -> tuple[str, int, str | None]:
- label, command, log_path = job
- sys.stdout.flush()
- sys.stderr.flush()
- if log_path is None:
- return label, run(command, dry_run=False, check=False), None
-
- saved_stdout = os.dup(1)
- saved_stderr = os.dup(2)
- try:
- with open(log_path, "w", encoding="utf-8", buffering=1) as log:
- os.dup2(log.fileno(), 1)
- os.dup2(log.fileno(), 2)
- with redirect_stdout(log), redirect_stderr(log):
- returncode = run(command, dry_run=False, check=False)
- finally:
- os.dup2(saved_stdout, 1)
- os.dup2(saved_stderr, 2)
- os.close(saved_stdout)
- os.close(saved_stderr)
- return label, returncode, log_path.read_text(encoding="utf-8", errors="replace")
-
-
def validate_pimcomp_source() -> None:
header = PIMCOMP_SOURCE / "backend/GeneticAlgorithm.h"
source = header.read_text(encoding="utf-8")
@@ -469,7 +438,7 @@ def comparison_command_for(
reuse_pimcomp_report = report
elif reuse_shared_pimcomp:
reuse_pimcomp_report = spec.shared_pimcomp_dir.parent / "comparison_report.json"
- return comparison_command(
+ command = comparison_command(
spec.model,
spec.functional_model,
spec.output_dir,
@@ -491,6 +460,7 @@ def comparison_command_for(
),
reuse_pimcomp_report=reuse_pimcomp_report,
)
+ return command + ["--jobs", "1", "--threads-per-job", str(args.threads_per_job)]
def pimcomp_report_stamp(output_dir: Path) -> tuple[int, int, int] | None:
@@ -506,40 +476,6 @@ def pimcomp_report_ready(output_dir: Path, previous: tuple[int, int, int] | None
return stamp is not None and stamp != previous
-def run_comparison_jobs(
- comparison_jobs: list[tuple[str, list[str], Path | None]],
- max_workers: int,
- log_dir: Path | None,
- log_offset: int,
-) -> tuple[list[str], int]:
- if not comparison_jobs:
- return [], log_offset
- print_directly = min(max_workers, len(comparison_jobs)) == 1
- jobs = [
- (
- label,
- command,
- None if print_directly or log_dir is None else log_dir / f"{log_offset + index}.log",
- )
- for index, (label, command, _) in enumerate(comparison_jobs)
- ]
- with (nullcontext(None) if print_directly else ProcessPoolExecutor(max_workers=max_workers)) as executor:
- completed = (
- map(run_comparison_job, jobs)
- if print_directly
- else (
- future.result()
- for future in as_completed(executor.submit(run_comparison_job, job) for job in jobs)
- )
- )
- failed = []
- for label, returncode, output in completed:
- print_completed(label, output or "")
- if returncode:
- failed.append(label)
- return failed, log_offset + len(jobs)
-
-
def ensure_throughput_config(arch: str, sim_time_ms: int) -> Path:
source = PIMCOMP_CONFIGS / arch / "throughput_config_1000ms.json"
target = PIMCOMP_CONFIGS / arch / f"throughput_config_{sim_time_ms}ms.json"
@@ -566,7 +502,7 @@ def config_path(arch: str, mode: str, sim_time_ms: int, *, write: bool) -> Path:
return path
-def main() -> int:
+def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Compare supported Pimcomp models with Raptor latency and throughput schedules."
)
@@ -635,9 +571,9 @@ def main() -> int:
parser.add_argument(
"-j",
"--jobs",
- type=int,
- default=4,
- help="Number of comparisons to run in parallel (default: 4).",
+ type=positive_integer,
+ default=available_processors(),
+ help="Maximum concurrent comparisons (default: available logical processors).",
)
parser.add_argument(
"--clean",
@@ -657,7 +593,11 @@ def main() -> int:
dest="raptor_extra_args",
help="Additional argument to pass to Raptor; repeat as needed.",
)
- args = parser.parse_args()
+ parser.add_argument("--threads-per-job", type=positive_integer, default=available_processors(),
+ help="Threads per compiler/simulator job (default: available logical processors).")
+ parser.add_argument("--with-ablation", action="store_true", help="Run comparison and ablation in one dependency queue.")
+ parser.add_argument("--ablation-out-dir", type=Path, help="Output root for a combined ablation run.")
+ args = parser.parse_args(argv)
if args.ablation_variant and Path(args.ablation_variant).name != args.ablation_variant:
parser.error("--ablation-variant must be a single directory name")
@@ -665,25 +605,35 @@ def main() -> int:
if args.only is not None and args.raptor_only:
parser.error("--only cannot be combined with --raptor-only")
+ return args
+
+
+@dataclass
+class SuitePlan:
+ args: argparse.Namespace
+ root: Path | None
+ common_root: Path | None
+ comparisons: dict
+ reference_batch_size: int
+ specs: list[ComparisonSpec]
+ aliases: dict[Path, Path]
+ previous_reports: dict[Path, tuple[int, int, int] | None]
+
+
+def make_plan(args: argparse.Namespace) -> SuitePlan:
out_dir = args.out_dir.resolve() if args.out_dir is not None else None
common_root = args.common_dir.resolve() if args.common_dir is not None else None
args.archs = list(dict.fromkeys(args.archs))
+ args.models = list(dict.fromkeys(args.models))
args.mode = list(dict.fromkeys(args.mode))
- if args.clean:
- print(
- f"Removed {clean_artifacts(out_dir, args.models, common_root)} "
- "comparison artifact path(s)."
- )
- return 0
-
if args.jobs < 1:
- parser.error("--jobs must be at least 1")
+ raise ValueError("--jobs must be at least 1")
if args.pimsim_time_ms <= 0:
- parser.error("--pimsim-time-ms must be positive")
+ raise ValueError("--pimsim-time-ms must be positive")
if args.batch_size <= 0:
- parser.error("--batch-size must be positive")
+ raise ValueError("--batch-size must be positive")
if args.timeout_seconds < 0:
- parser.error("--timeout-seconds must be non-negative")
+ raise ValueError("--timeout-seconds must be non-negative")
comparisons_by_arch: dict[str, tuple[tuple[str, int | str, str], ...]] = {}
configs_by_arch: dict[str, dict[str, Path]] = {}
for arch in args.archs:
@@ -697,7 +647,7 @@ def main() -> int:
and (args.pipeline is None or comparison[1] == args.pipeline)
)
if not comparisons:
- parser.error("--pipeline does not belong to the selected --mode")
+ raise ValueError("--pipeline does not belong to the selected --mode")
configs = {
mode: config_path(arch, mode, args.pimsim_time_ms, write=not args.dry_run)
for mode, _, _ in comparisons
@@ -716,7 +666,7 @@ def main() -> int:
if not MODELS[name].exists()
]
if missing:
- parser.error(f"missing model(s): {', '.join(missing)}")
+ raise ValueError(f"missing model(s): {', '.join(missing)}")
if args.only is not None:
missing_reuse = []
@@ -730,7 +680,7 @@ def main() -> int:
if not required.exists():
missing_reuse.append(str(required))
if missing_reuse:
- parser.error(
+ raise ValueError(
f"--only {args.only} needs existing comparison artifacts: "
+ ", ".join(missing_reuse)
)
@@ -740,41 +690,6 @@ def main() -> int:
if out_dir is not None and not args.dry_run:
out_dir.mkdir(parents=True, exist_ok=True)
- print(Style.BRIGHT + f"Found {len(args.models)} Pimcomp model(s) to compare." + Style.RESET_ALL)
- print(f"Architectures: {', '.join(args.archs)}")
- print(f"Modes: {', '.join(args.mode)}")
- print(f"Throughput Pimsim time: {args.pimsim_time_ms} ms")
- print(f"Max parallel jobs: {args.jobs}")
- print(f"Shared reference batch: {reference_batch_size}")
- print(
- f"Comparison jobs: "
- f"{sum(len(args.models) * len(comparisons) for comparisons in comparisons_by_arch.values())}"
- )
- print(f"Artifacts root: {out_dir or SUITE}")
- print("=" * 72)
-
- print_stage("Prepare shared artifacts", STAGE_COLORS["Build runner"])
- try:
- for name in args.models:
- try:
- run(
- prepare_common_command(
- MODELS[name],
- common_dir(out_dir, name, common_root),
- args.timeout_seconds,
- reference_batch_size,
- args.jobs,
- ),
- dry_run=args.dry_run,
- )
- finally:
- print_completed(f"shared artifacts: {name}")
- except KeyboardInterrupt:
- remove_lock_files(out_dir or SUITE)
- print("Interrupted; cleaned validation lock files.", file=sys.stderr)
- return 130
-
- failed = []
comparison_specs: list[ComparisonSpec] = []
shared_pimcomp_by_group: dict[tuple[str, str, str], Path] = {}
for arch in args.archs:
@@ -808,117 +723,171 @@ def main() -> int:
)
)
- if args.dry_run:
- for spec in comparison_specs:
- command = comparison_command_for(
- spec,
- args,
- reuse_shared_pimcomp=not spec.anchor,
- )
- if run(command, dry_run=True, check=False):
+ return SuitePlan(args, out_dir, common_root, comparisons_by_arch,
+ reference_batch_size, comparison_specs, {}, {})
+
+
+def execution_key(plan: SuitePlan, spec: ComparisonSpec) -> tuple:
+ args = plan.args
+ return (spec.model, spec.common_dir, spec.config, spec.mode, spec.pipeline,
+ tuple(args.raptor_extra_args), args.batch_size, args.no_fast,
+ args.threads_per_job)
+
+
+def build_tasks(plans: list[SuitePlan]) -> list[ExperimentTask]:
+ tasks = []
+ references = {}
+ comparison_sources = {execution_key(plan, spec): spec.output_dir
+ for plan in plans if not plan.args.ablation_variant and plan.args.only is None
+ and not plan.args.raptor_only for spec in plan.specs}
+ for plan in plans:
+ for name in plan.args.models:
+ common = common_dir(plan.root, name, plan.common_root)
+ previous = references.get(common)
+ if previous is None or previous[0] < plan.reference_batch_size:
+ references[common] = (plan.reference_batch_size, name, plan.args)
+ for common, (batch_size, name, args) in references.items():
+ command = prepare_common_command(MODELS[name], common, args.timeout_seconds,
+ batch_size, args.threads_per_job)
+ command += ["--threads-per-job", str(args.threads_per_job)]
+ tasks.append(ExperimentTask("reference:" + str(common), lambda command=command: command,
+ (), common / "prepare.log"))
+ for plan in plans:
+ args = plan.args
+ for spec in plan.specs:
+ source = comparison_sources.get(execution_key(plan, spec)) if args.ablation_variant == "baseline" else None
+ report_root = source if source is not None else spec.output_dir
+ plan.previous_reports[spec.output_dir] = pimcomp_report_stamp(report_root / "pimcomp/output")
+ if args.ablation_variant == "baseline" and execution_key(plan, spec) in comparison_sources:
+ plan.aliases[spec.output_dir] = comparison_sources[execution_key(plan, spec)]
+ continue
+ reuse = not (spec.anchor or args.only == "raptor" or args.raptor_only)
+ dependencies = ["reference:" + str(spec.common_dir)]
+ previous_report = pimcomp_report_stamp(spec.shared_pimcomp_dir) if reuse else None
+ if reuse:
+ dependencies.append(str(spec.shared_pimcomp_dir.parent.parent))
+
+ def command(spec=spec, args=args, reuse=reuse, previous=previous_report):
+ if reuse and not args.dry_run and not pimcomp_report_ready(spec.shared_pimcomp_dir, previous):
+ raise ValueError(f"Pimcomp prerequisite produced no fresh report: {spec.shared_pimcomp_dir}")
+ return comparison_command_for(spec, args, reuse_shared_pimcomp=reuse)
+
+ tasks.append(ExperimentTask(str(spec.output_dir), command, tuple(dependencies),
+ spec.output_dir / "comparison.log"))
+ return tasks
+
+
+def report_for(plan: SuitePlan, spec: ComparisonSpec) -> Path:
+ return plan.aliases.get(spec.output_dir, spec.output_dir) / "pimcomp/comparison_report.json"
+
+
+def finish_plan(plan: SuitePlan, results: dict[str, TaskResult]) -> int:
+ args = plan.args
+ failed, diagnostic = [], []
+ overrides = {}
+ for spec in plan.specs:
+ directory = plan.aliases.get(spec.output_dir, spec.output_dir)
+ result = results.get(str(directory))
+ fresh = pimcomp_report_ready(directory / "pimcomp/output", plan.previous_reports[spec.output_dir])
+ overrides[spec.output_dir] = report_for(plan, spec) if result and not result.returncode and fresh else None
+ for arch in args.archs:
+ output = write_results_csv(plan.root, arch, args.models, plan.comparisons[arch],
+ args.ablation_variant, overrides, args.raptor_only)
+ if plan.aliases:
+ destination = (plan.root or SUITE) / "ablation" / args.ablation_variant / "source_reports.json"
+ destination.write_text(json.dumps({str(k): str(v) if v is not None else None
+ for k, v in overrides.items()}, indent=2) + "\n")
+ for spec in plan.specs:
+ result = results.get(str(plan.aliases.get(spec.output_dir, spec.output_dir)))
+ path = report_for(plan, spec)
+ report = json.loads(path.read_text()) if overrides[spec.output_dir] is not None else {}
+ if result is None or result.returncode or overrides[spec.output_dir] is None:
+ failed.append(spec.label)
+ elif not comparison_passed(report):
+ if args.ablation_variant == "no-sync" and completed_synchronization_ablation(report):
+ diagnostic.append(spec.label)
+ else:
failed.append(spec.label)
- elif comparison_specs:
- raptor_only_run = args.only == "raptor" or args.raptor_only
- anchor_specs = comparison_specs if raptor_only_run else [
- spec for spec in comparison_specs if spec.anchor
- ]
- anchor_jobs = [
- (
- spec.label,
- comparison_command_for(spec, args, reuse_shared_pimcomp=False),
- None,
- )
- for spec in anchor_specs
- ]
- dependent_specs = [] if raptor_only_run else [
- spec for spec in comparison_specs if not spec.anchor
- ]
- print_directly = min(args.jobs, len(comparison_specs)) == 1
- try:
- with (nullcontext(None) if print_directly else TemporaryDirectory(prefix="raptor-pimcomp-")) as log_dir:
- previous_reports = {
- spec.shared_pimcomp_dir: pimcomp_report_stamp(spec.shared_pimcomp_dir)
- for spec in anchor_specs
- }
- anchor_failed, log_offset = run_comparison_jobs(
- anchor_jobs,
- args.jobs,
- Path(log_dir) if log_dir else None,
- 0,
- )
- failed.extend(anchor_failed)
- dependent_jobs = [
- (
- spec.label,
- comparison_command_for(
- spec,
- args,
- reuse_shared_pimcomp=pimcomp_report_ready(
- spec.shared_pimcomp_dir, previous_reports[spec.shared_pimcomp_dir]
- ),
- ),
- None,
- )
- for spec in dependent_specs
- ]
- dependent_failed, _ = run_comparison_jobs(
- dependent_jobs,
- args.jobs,
- Path(log_dir) if log_dir else None,
- log_offset,
- )
- failed.extend(dependent_failed)
- except KeyboardInterrupt:
- remove_lock_files(out_dir or SUITE)
- print("Interrupted; cleaned validation lock files.", file=sys.stderr)
- return 130
-
- if args.dry_run:
- return 1 if failed else 0
-
- remove_lock_files(out_dir or SUITE)
-
- results_path = None
- for arch in args.archs:
- results_path = write_results_csv(
- out_dir,
- arch,
- args.models,
- comparisons_by_arch[arch],
- args.ablation_variant,
- )
- assert results_path is not None
- print_stage(results_path.name, STAGE_COLORS["Compare outputs"])
- print(results_path.read_text(encoding="utf-8"), end="")
- diagnostic_failures = []
- for arch in args.archs:
- for name in args.models:
- for mode, pipeline, _ in comparisons_by_arch[arch]:
- label = f"{arch}/{name}/{mode}/pipeline{pipeline}"
- report_path = result_dir(
- out_dir, name, arch, mode, pipeline, args.ablation_variant
- ) / "pimcomp/comparison_report.json"
- report = json.loads(report_path.read_text(encoding="utf-8")) if report_path.exists() else {}
- if not comparison_passed(report):
- if (label not in failed and args.ablation_variant == "no-sync"
- and completed_synchronization_ablation(report)):
- diagnostic_failures.append(label)
- elif label not in failed:
- failed.append(label)
- print(Style.BRIGHT + Fore.CYAN + "Summary" + Style.RESET_ALL)
- total_jobs = sum(len(args.models) * len(comparisons) for comparisons in comparisons_by_arch.values())
- print(Style.BRIGHT + f"Passed: {total_jobs - len(failed) - len(diagnostic_failures)}" + Style.RESET_ALL)
- print(Style.BRIGHT + f"Failed: {len(failed)}" + Style.RESET_ALL)
- if diagnostic_failures:
- print("No-sync numerical failures (diagnostic only): " + ", ".join(diagnostic_failures))
- print(Style.BRIGHT + f"Results: {results_path}" + Style.RESET_ALL)
+ print(f"Results: {output}; passed: {len(plan.specs) - len(failed) - len(diagnostic)}; "
+ f"failed: {len(failed)}; diagnostic no-sync failures: {len(diagnostic)}", flush=True)
if failed:
- print(
- Fore.RED + f"Failed comparisons: {', '.join(failed)}" + Style.RESET_ALL,
- file=sys.stderr,
- )
- return 1
- return 0
+ print("Failed comparisons: " + ", ".join(failed), file=sys.stderr)
+ remove_lock_files(plan.root or SUITE)
+ return int(bool(failed))
+
+
+def run_plans(plans: list[SuitePlan]) -> int:
+ tasks = build_tasks(plans)
+ args = plans[0].args
+ if any((plan.args.jobs, plan.args.threads_per_job) != (args.jobs, args.threads_per_job) for plan in plans):
+ raise ValueError("all experiment plans must use the same job and thread limits")
+ if args.dry_run:
+ for task in tasks:
+ run(task.command(), dry_run=True, check=False)
+ return 0
+ print(f"Concurrent jobs: {args.jobs}; threads per job: {args.threads_per_job}", flush=True)
+ results = run_tasks(tasks, args.jobs, args.threads_per_job, REPO)
+ # Only the parent writes aggregate CSVs.
+ failures = [finish_plan(plan, results) for plan in plans]
+ return int(any(failures))
+
+
+def combined_ablation_plans(args: argparse.Namespace) -> tuple[list[SuitePlan], object, Path]:
+ sys.path.insert(0, str(REPO / "validation/tools/pim/ablation"))
+ import run_ablation as ablation
+ from copy import deepcopy
+ root = args.ablation_out_dir or args.out_dir or SUITE
+ models = [name for name in args.models if name in ablation.ABLATION_DEFAULT_MODELS]
+ if not models:
+ raise ValueError("combined ablation needs at least one of the four ablation models")
+ cases = [(mode, depth) for mode, depth, _ in COMPARISONS
+ if mode in args.mode and (depth in (1, 4) if args.pipeline is None else depth == args.pipeline)]
+ if args.pipeline is not None and not cases:
+ cases = [("throughput", args.pipeline)]
+ plans = []
+ for variant in ablation.VARIANTS:
+ for mode, depth in cases:
+ if variant.name == "no-sync" and mode != "throughput":
+ continue
+ selected = deepcopy(args)
+ selected.out_dir, selected.models = root, models
+ selected.common_dir = args.common_dir or args.out_dir or SUITE
+ selected.raptor_only, selected.only = True, None
+ selected.ablation_variant = variant.name
+ selected.mode, selected.pipeline = [mode], depth
+ selected.raptor_extra_args += list(variant.raptor_args)
+ plans.append(make_plan(selected))
+ return plans, ablation, root
+
+
+def main(argv: list[str] | None = None) -> int:
+ args = parse_args(argv)
+ if args.clean:
+ print(f"Removed {clean_artifacts(args.out_dir, args.models, args.common_dir)} comparison artifact path(s).")
+ return 0
+ try:
+ plans = [make_plan(args)]
+ if args.with_ablation:
+ if args.only or args.raptor_only or args.ablation_variant:
+ raise ValueError("--with-ablation requires a full compiler comparison")
+ extra, ablation, ablation_root = combined_ablation_plans(args)
+ plans.extend(extra)
+ result = run_plans(plans)
+ if args.with_ablation and not args.dry_run:
+ cases = frozenset((spec.mode, str(spec.pipeline)) for plan in extra for spec in plan.specs)
+ variants = [variant for variant in ablation.VARIANTS
+ if any(plan.args.ablation_variant == variant.name for plan in extra)]
+ _, failures = ablation.aggregate_results(ablation_root, variants, cases,
+ frozenset(extra[0].args.models),
+ output=ablation.DEFAULT_RESULTS_PATH if args.out_dir is None and args.ablation_out_dir is None else None)
+ result = int(bool(result or failures))
+ return result
+ except KeyboardInterrupt:
+ print("Interrupted; terminated experiment workers.", file=sys.stderr)
+ return 130
+ except ValueError as exc:
+ print(str(exc), file=sys.stderr)
+ return 2
if __name__ == "__main__":