Merge remote-tracking branch 'origin/main'
Validate Operations / validate-operations (push) Has been cancelled

This commit is contained in:
NiccoloN
2026-11-05 15:54:00 +01:00
21 changed files with 1161 additions and 311 deletions
+3
View File
@@ -73,6 +73,9 @@ differ, and validation defaults to Arch-A.
## Documentation
The [paper artifact](artifact/README.md) builds the pinned compiler and simulators
in Docker, runs the comparison and ablation experiments, and generates plots.
Start with the [documentation index](docs/README.md). It links the
[compiler architecture](docs/compiler/architecture.md),
[compiler options](docs/compiler/options.md), and
+13
View File
@@ -0,0 +1,13 @@
*
!Dockerfile
!pins.env
!requirements.lock
!models.sha256
!build.sh
!run.py
!plot_ablation.py
!source.tar.gz
!source.sha256
!validation-overlay/
!validation-overlay/**
!validation-overlay.sha256
+7
View File
@@ -0,0 +1,7 @@
/source.tar.gz
/source.tar.gz.tmp
/source.sha256
/dist/
/results/
/validation-overlay/
/validation-overlay.sha256
+46
View File
@@ -0,0 +1,46 @@
FROM python:3.12.12-slim-trixie AS python-deps
ENV DEBIAN_FRONTEND=noninteractive \
CC=clang-19 CXX=clang++-19 \
LDFLAGS=-fuse-ld=mold \
RUSTFLAGS="-C linker=clang-19 -C link-arg=-fuse-ld=mold" \
CARGO_HOME=/opt/cargo RUSTUP_HOME=/opt/rustup \
PATH=/opt/raptor/.venv/bin:/opt/cargo/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 MPLBACKEND=Agg \
PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 \
HOME=/tmp XDG_CACHE_HOME=/tmp/cache MPLCONFIGDIR=/tmp/matplotlib
RUN apt-get update && apt-get install -y --no-install-recommends \
build-essential ca-certificates clang-19 curl git libomp-19-dev \
libzstd-dev mold ninja-build pkg-config zlib1g-dev \
&& rm -rf /var/lib/apt/lists/*
WORKDIR /opt/artifact
COPY pins.env requirements.lock models.sha256 build.sh ./
RUN python -m venv /opt/raptor/.venv \
&& /opt/raptor/.venv/bin/python -m pip install --no-cache-dir --only-binary=onnx -r requirements.lock
FROM python-deps AS artifact
ARG BUILD_JOBS
RUN . ./pins.env && curl --proto '=https' --tlsv1.2 -fsSL https://sh.rustup.rs -o /tmp/rustup.sh \
&& sh /tmp/rustup.sh -y --profile minimal --default-toolchain "$RUST_VERSION" \
&& rm /tmp/rustup.sh
COPY source.tar.gz source.sha256 ./
RUN sha256sum -c source.sha256 \
&& . ./pins.env \
&& gzip -dc source.tar.gz > /tmp/raptor-source.tar \
&& test "$(git get-tar-commit-id < /tmp/raptor-source.tar)" = "$RAPTOR_COMMIT" \
&& tar -xf /tmp/raptor-source.tar -C /opt/raptor \
&& rm source.tar.gz /tmp/raptor-source.tar
RUN bash build.sh "${BUILD_JOBS:-$(env -u OMP_NUM_THREADS -u OMP_THREAD_LIMIT nproc)}"
COPY validation-overlay/ ./validation-overlay/
COPY validation-overlay.sha256 ./
RUN sha256sum -c validation-overlay.sha256 \
&& cp -R validation-overlay/validation/. /opt/raptor/validation/ \
&& cp validation-overlay.sha256 /opt/artifact/provenance/
COPY run.py plot_ablation.py ./
WORKDIR /opt/raptor
ENTRYPOINT ["/opt/raptor/.venv/bin/python", "/opt/artifact/run.py"]
CMD ["reproduce"]
+32
View File
@@ -0,0 +1,32 @@
# Raptor paper artifact
Raptor is fixed to commit `4a1fe710b2f0e4e9c0504f4fd9b4c4dd3df53b7d`.
Requires Docker on Linux or WSL2. The build downloads dependencies and uses Clang 19 and mold.
Install the host dependencies:
```bash
sudo apt install -y docker.io docker-cli docker-buildx git
sudo systemctl enable --now docker
sudo usermod -aG docker "$USER"
newgrp docker
```
Log out and back in, then run from this directory:
```bash
bash artifact.sh build
bash artifact.sh reproduce
```
Open `results/full/index.html` to view the plots and result tables.
The page also links to the CSV files and logs. Failed numerical validation remains visible in the results.
For a small VGG8 run, use `bash artifact.sh smoke` and open `results/smoke/index.html`.
Build jobs, run jobs, and threads per job default to the available logical processor count.
Override with `BUILD_JOBS`, `JOBS`, and `THREADS_PER_JOB`, or run `bash artifact.sh reproduce --jobs 8 --threads-per-job 4`.
For another run, set `OUTPUT_DIR` to a new directory.
To prepare the paper attachment, run `bash artifact/artifact.sh package` from the repository root.
The archive is written to `artifact/dist/` and includes the pinned sources and models.
Updated scheduling code is included separately; compiler and simulator sources stay pinned.
+61
View File
@@ -0,0 +1,61 @@
#!/usr/bin/env bash
set -euo pipefail
cd -- "$(dirname -- "${BASH_SOURCE[0]}")"
source pins.env
image="raptor-artifact:${RAPTOR_COMMIT:0:12}"
command=${1:-help}
if (($#)); then shift; fi
case "$command" in
prepare)
exec bash prepare.sh
;;
package)
bash prepare.sh
mkdir -p dist
archive="raptor-artifact-${RAPTOR_COMMIT:0:12}.tar.gz"
package_dir=$(mktemp -d)
trap 'rm -rf -- "$package_dir"' EXIT
mkdir "$package_dir/raptor-artifact"
cp \
README.md artifact.sh prepare.sh Dockerfile .dockerignore pins.env \
requirements.lock models.sha256 build.sh run.py plot_ablation.py source.tar.gz source.sha256 \
"$package_dir/raptor-artifact/"
cp -R validation-overlay "$package_dir/raptor-artifact/"
cp validation-overlay.sha256 "$package_dir/raptor-artifact/"
tar -czf "dist/$archive" -C "$package_dir" raptor-artifact
(cd dist && sha256sum "$archive" > "$archive.sha256")
printf 'Attachment: %s/dist/%s\n' "$PWD" "$archive"
;;
build)
[[ -f source.tar.gz ]] || bash prepare.sh
if [[ -f ../validation/raptor_validation/experiment_scheduler.py ]]; then
bash prepare.sh --overlay-only
fi
sha256sum -c source.sha256
docker build --build-arg "BUILD_JOBS=${BUILD_JOBS:-$(env -u OMP_NUM_THREADS -u OMP_THREAD_LIMIT nproc)}" -t "$image" .
;;
reproduce|smoke|plots)
output=${OUTPUT_DIR:-"$PWD/results"}
mkdir -p "$output"
output=$(cd -- "$output" && pwd)
image_id=$(docker image inspect --format '{{.Id}}' "$image")
status=0
docker run --rm --init --network none \
--user "$(id -u):$(id -g)" \
--env "ARTIFACT_IMAGE_ID=$image_id" \
--env "JOBS=${JOBS:-$(env -u OMP_NUM_THREADS -u OMP_THREAD_LIMIT nproc)}" \
--env "THREADS_PER_JOB=${THREADS_PER_JOB:-$(env -u OMP_NUM_THREADS -u OMP_THREAD_LIMIT nproc)}" \
--mount "type=bind,src=$output,dst=/results" \
"$image" "$command" "$@" || status=$?
profile=full
if [[ "$command" = smoke || " $* " = *' --smoke '* ]]; then profile=smoke; fi
if [[ -f "$output/$profile/index.html" ]]; then
printf '\nOpen %s/%s/index.html\n' "$output" "$profile"
fi
exit "$status"
;;
*)
echo 'Usage: bash artifact.sh {build|smoke|reproduce|plots|package}'
[[ "$command" = help ]]
;;
esac
+75
View File
@@ -0,0 +1,75 @@
#!/usr/bin/env bash
set -euo pipefail
source /opt/artifact/pins.env
export CMAKE_BUILD_PARALLEL_LEVEL=${1:-$(env -u OMP_NUM_THREADS -u OMP_THREAD_LIMIT nproc)}
[[ "$CMAKE_BUILD_PARALLEL_LEVEL" =~ ^[1-9][0-9]*$ ]]
cd /opt/raptor
sha256sum -c /opt/artifact/models.sha256
checkout() {
local url=$1 revision=$2 destination=$3
git init "$destination"
git -C "$destination" remote add origin "$url"
git -C "$destination" fetch --depth 1 origin "$revision"
git -C "$destination" checkout --detach FETCH_HEAD
test "$(git -C "$destination" rev-parse HEAD)" = "$revision"
git -C "$destination" submodule update --init --recursive --depth 1
}
checkout https://github.com/onnx/onnx-mlir.git "$ONNX_MLIR_COMMIT" onnx-mlir
checkout https://github.com/llvm/llvm-project.git "$LLVM_COMMIT" onnx-mlir/llvm-project
checkout https://github.com/HEAPLab/PIMCOMP-NN.git "$PIMCOMP_COMMIT" third_party/PIMCOMP-NN
checkout https://github.com/HEAPLab/pimsim-nn.git "$PIMSIM_COMMIT" backend-simulators/pim/pimsim-nn
checkout https://github.com/protocolbuffers/protobuf.git "$PROTOBUF_COMMIT" /opt/protobuf
common=(-G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_POSITION_INDEPENDENT_CODE=ON
-DCMAKE_EXE_LINKER_FLAGS=-fuse-ld=mold
-DCMAKE_SHARED_LINKER_FLAGS=-fuse-ld=mold
-DCMAKE_MODULE_LINKER_FLAGS=-fuse-ld=mold)
cmake -S /opt/protobuf -B /opt/protobuf/build "${common[@]}" \
-Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_ABSL_PROVIDER=module \
-DABSL_PROPAGATE_CXX_STD=ON -DCMAKE_CXX_STANDARD=17
cmake --build /opt/protobuf/build
cmake --install /opt/protobuf/build
ldconfig
llvm=onnx-mlir/llvm-project
cmake -S "$llvm/llvm" -B "$llvm/build_release" "${common[@]}" \
'-DLLVM_ENABLE_PROJECTS=mlir;clang' -DLLVM_ENABLE_RUNTIMES=openmp \
-DLLVM_TARGETS_TO_BUILD=host -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_ENABLE_RTTI=ON \
-DENABLE_LIBOMPTARGET=OFF -DLLVM_ENABLE_LIBEDIT=OFF \
-DLLVM_PARALLEL_LINK_JOBS="$CMAKE_BUILD_PARALLEL_LEVEL"
cmake --build "$llvm/build_release"
cmake -S . -B build_release "${common[@]}" \
-DONNX_MLIR_ACCELERATORS=PIM -DLLVM_ENABLE_ASSERTIONS=ON \
-DONNX_MLIR_ENABLE_JAVA=OFF -DONNX_MLIR_BUILD_TESTS=OFF \
-DMLIR_DIR="$PWD/$llvm/build_release/lib/cmake/mlir"
cmake --build ./build_release
cmake -S third_party/PIMCOMP-NN -B third_party/PIMCOMP-NN/build "${common[@]}"
cmake --build third_party/PIMCOMP-NN/build --target PIMCOMP-NN
sim=backend-simulators/pim/pimsim-nn
cmake -S "$sim" -B "$sim/build" "${common[@]}"
cmake --build "$sim/build"
ctest --test-dir "$sim/build" --output-on-failure
crate=backend-simulators/pim/pim-simulator
cargo build --locked --release --no-default-features --manifest-path "$crate/Cargo.toml" \
--package pim-simulator --bin pim-simulator --jobs "$CMAKE_BUILD_PARALLEL_LEVEL"
# The experiment driver uses cargo run. All dependencies must be available offline.
cargo fetch --locked --manifest-path "$crate/Cargo.toml"
mkdir -p /opt/artifact/provenance
cp /opt/artifact/pins.env /opt/artifact/source.sha256 /opt/artifact/models.sha256 /opt/artifact/provenance/
for dir in onnx-mlir "$llvm" third_party/PIMCOMP-NN "$sim" /opt/protobuf; do
printf '%s %s\n' "$dir" "$(git -C "$dir" rev-parse HEAD)"
git -C "$dir" submodule status --recursive
done > /opt/artifact/provenance/revisions.txt
dpkg-query -W > /opt/artifact/provenance/system-packages.txt
.venv/bin/python -m pip freeze > /opt/artifact/provenance/python-packages.txt
{ clang-19 --version; mold --version; cmake --version; rustc --version; } \
> /opt/artifact/provenance/toolchain.txt
# Runtime uses an arbitrary host UID. Cargo and the runners need writable caches
# and ONNX-MLIR writes temporary shims/configurations inside its source tree.
chmod -R a+rwX /opt/raptor /opt/cargo /opt/rustup
+5
View File
@@ -0,0 +1,5 @@
788088b908e233d924c7c26b997e89ee861290c7bc56783a306e8201d79aac8f validation/networks/pimcomp_models/resnet18/resnet18-v1-7.onnx
c3231061d081bdd47884137b02134f85142752a39e87263c529cd14ed242b096 validation/networks/pimcomp_models/resnet34/resnet34-v1-7.onnx
c960c195953423388733cd0e60712cf425ca1f8917e264f5cf98e028752c61cf validation/networks/pimcomp_models/googlenet/googlenet-12-no-lrn.onnx
396cdea21e5e7d02c3f26f14d22ef20975171702493f5c5e79b8e0d896e541ef validation/networks/pimcomp_models/vgg8/vgg8-mnist-reconstructed.onnx
5e7bd85dc60b252674f8d7e1ad24f2317d00dfb8c524826536698462de04620d validation/networks/pimcomp_models/yolo11n/yolo11n.onnx
+8
View File
@@ -0,0 +1,8 @@
# Raptor main at artifact preparation time. Do not replace this with a branch.
RAPTOR_COMMIT=4a1fe710b2f0e4e9c0504f4fd9b4c4dd3df53b7d
ONNX_MLIR_COMMIT=eb54c2afc46d00c6b196d1f275b6bfee17e12f69
LLVM_COMMIT=0c2701fe7fa002e1befc5f86c268a7964f96d286
PIMCOMP_COMMIT=7b6be6da4d33bf19f68d073eccac2a78e326a5af
PIMSIM_COMMIT=17e37c15da5473362e53268dabdd07dff5fe8fbf
PROTOBUF_COMMIT=6a6cd88c262ffdb1738167a47d5fcc7a3eb4edac
RUST_VERSION=1.90.0
+59
View File
@@ -0,0 +1,59 @@
"""Plot planning and synchronization gains without hiding invalid programs."""
from __future__ import annotations
import argparse
import csv
import math
from pathlib import Path
import matplotlib
matplotlib.use('Agg')
import matplotlib.pyplot as plt
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('--input', type=Path, required=True)
parser.add_argument('--output-dir', type=Path, required=True)
args = parser.parse_args()
with args.input.open(newline='') as stream:
rows = list(csv.DictReader(stream))
if not rows:
raise ValueError('The ablation CSV is empty')
panels = [('plan', 'latency', 'latency_gain_percent', 'Planning: latency reduction (%)'),
('plan', 'throughput', 'throughput_gain_percent', 'Planning: throughput increase (%)'),
('sync', 'throughput', 'throughput_gain_percent', 'Synchronization: throughput increase (%)')]
fig, axes = plt.subplots(3, 1, figsize=(11, 9), layout='constrained')
for ax, (comparison, mode, field, label) in zip(axes, panels):
selected = sorted((r for r in rows if r['comparison'] == comparison and r['mode'] == mode),
key=lambda r: (r['arch'], r['model'], int(r['raptor_pipeline'])))
for x, row in enumerate(selected):
try:
value = float(row[field].rstrip('%'))
except (ValueError, TypeError):
value = math.nan
valid = row['functional_validation'] == row['baseline_functional_validation'] == 'PASS'
if math.isfinite(value):
ax.bar(x, value, color='#B3D5CA' if valid else '#FF9494',
edgecolor='#222222', hatch='' if valid else '//')
ax.annotate(f'{value:.1f}' + ('' if valid else ' INVALID'),
(x, value), xytext=(0, 4 if value >= 0 else -4),
textcoords='offset points', ha='center',
va='bottom' if value >= 0 else 'top', fontsize=7)
else:
ax.text(x, 0, 'NA', ha='center', va='bottom', fontsize=8)
ax.set_xticks(range(len(selected)),
[f"{r['model']}\n{r['arch']} p{r['raptor_pipeline']}" for r in selected], fontsize=8)
ax.set_ylabel(label)
ax.axhline(0, color='#222222', linewidth=.7)
ax.set_xlim(-.6, max(.6, len(selected) - .4))
ax.margins(y=.3)
fig.suptitle('Raptor ablation (red hatching: functional validation failed)')
args.output_dir.mkdir(parents=True, exist_ok=True)
for extension in ('pdf', 'svg', 'png'):
fig.savefig(args.output_dir / f'ablation.{extension}', dpi=200)
plt.close(fig)
if __name__ == '__main__':
main()
+39
View File
@@ -0,0 +1,39 @@
#!/usr/bin/env bash
set -euo pipefail
cd -- "$(dirname -- "${BASH_SOURCE[0]}")"
source pins.env
repo=$(git -C .. rev-parse --show-toplevel)
test "$(git -C "$repo" rev-parse "$RAPTOR_COMMIT^{commit}")" = "$RAPTOR_COMMIT"
for item in "onnx-mlir:$ONNX_MLIR_COMMIT" \
"third_party/PIMCOMP-NN:$PIMCOMP_COMMIT" \
"backend-simulators/pim/pimsim-nn:$PIMSIM_COMMIT"; do
path=${item%:*}
expected=${item##*:}
actual=$(git -C "$repo" rev-parse "$RAPTOR_COMMIT:$path")
test "$actual" = "$expected"
done
if [[ "${1:-}" != --overlay-only ]]; then
# Include the complete compiler and evaluation code, plus only the suite models.
# The last two paths are the targets of the suite's relative model symlinks.
git -C "$repo" archive --format=tar "$RAPTOR_COMMIT" \
CMakeLists.txt README.md .gitmodules src test docs backend-simulators \
validation/README.md validation/validate.py validation/raptor_validation \
validation/tools validation/pimsim_configs validation/networks/pimcomp_models \
validation/networks/resnet18/depth_68/resnet18_depth_68.onnx \
validation/networks/yolo11n/depth_51/yolo11n_depth_51.onnx \
| gzip -n > source.tar.gz.tmp
mv source.tar.gz.tmp source.tar.gz
sha256sum source.tar.gz > source.sha256
fi
overlay=(validation/raptor_validation/experiment_scheduler.py
validation/tools/pim/pimcomp/compare/run_pimcomp_models.py
validation/tools/pim/pimcomp/compare/compare_raptor_pimcomp_model.py
validation/tools/pim/ablation/run_ablation.py
validation/tools/pim/pimcomp/compare/README.md
validation/tools/pim/ablation/README.md)
for path in "${overlay[@]}"; do
mkdir -p "validation-overlay/$(dirname "$path")"
cp "$repo/$path" "validation-overlay/$path"
done
sha256sum "${overlay[@]/#/validation-overlay/}" > validation-overlay.sha256
printf 'Prepared Raptor %s\n' "$RAPTOR_COMMIT"
+23
View File
@@ -0,0 +1,23 @@
# Evaluation and ONNX-MLIR build dependencies. The Graph Explorer is not required.
cmake==3.31.6
colorama==0.4.6
contourpy==1.3.2
cycler==0.12.1
execnet==2.1.1
fonttools==4.58.0
iniconfig==2.1.0
kiwisolver==1.4.8
lit==15.0.7
matplotlib==3.10.3
numpy==2.2.6
onnx==1.17.0
packaging==25.0
pillow==11.2.1
pluggy==1.5.0
protobuf==7.34.0
pyparsing==3.2.3
pytest==8.3.5
pytest-xdist==3.6.1
python-dateutil==2.9.0.post0
PyYAML==6.0.2
six==1.17.0
+140
View File
@@ -0,0 +1,140 @@
"""Container commands for the pinned Raptor paper artifact."""
from __future__ import annotations
import argparse
import csv
from datetime import datetime, timezone
from html import escape
import json
import os
from pathlib import Path
import platform
import shutil
import subprocess
import sys
import time
HERE = Path(__file__).resolve().parent
REPO = Path('/opt/raptor')
PYTHON = REPO / '.venv/bin/python'
COMPARE = Path('validation/tools/pim/pimcomp/compare')
ABLATION = Path('validation/tools/pim/ablation/run_ablation.py')
def stage(name: str, command: list[str], root: Path) -> dict:
print(f'[{name}] {command}', flush=True)
start = time.monotonic()
with (root / f'{name}.log').open('wb') as log:
with subprocess.Popen(command, cwd=REPO, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT) as process:
for line in process.stdout:
sys.stdout.buffer.write(line)
sys.stdout.buffer.flush()
log.write(line)
returncode = process.wait()
return {'command': command, 'exit_code': returncode,
'seconds': round(time.monotonic() - start, 3)}
def plots(root: Path, smoke: bool) -> dict:
commands = {'ablation-plots': [str(PYTHON), str(HERE / 'plot_ablation.py'),
'--input', str(root / 'ablation/results_ablation.csv'),
'--output-dir', str(root / 'plots/ablation')]}
if not smoke:
commands['comparison-plots'] = [str(PYTHON), str(COMPARE / 'generate_plots.py'),
'--input', str(root / 'comparison/results_comparison.csv'),
'--output-dir', str(root / 'plots/comparison')]
return {name: stage(name, command, root) for name, command in commands.items()}
def show_results(root: Path) -> None:
page = ['<!doctype html><html lang="en"><meta charset="utf-8">',
'<meta name="viewport" content="width=device-width, initial-scale=1">',
'<title>Raptor results</title><style>',
'body{font:16px sans-serif;max-width:1100px;margin:2rem auto;padding:0 1rem}',
'img{max-width:100%}table{border-collapse:collapse;font-size:13px}',
'th,td{padding:6px;border:1px solid #ccc;white-space:nowrap}',
'.table{overflow-x:auto}</style><h1>Raptor results</h1>',
'<p>Functional validation status is shown in the tables and plots.</p>']
for title, relative in (
('Comparison', 'plots/comparison/png/performance_energy_compact.png'),
('Pipeline throughput', 'plots/comparison/png/raptor_pipeline_throughput.png'),
('Ablation', 'plots/ablation/ablation.png'),
):
if (root / relative).is_file():
page.append(f'<h2>{title}</h2><a href="{relative}"><img src="{relative}" alt="{title}"></a>')
for title, relative in (('Comparison', 'comparison/results_comparison.csv'),
('Ablation', 'ablation/results_ablation.csv')):
path = root / relative
if path.is_file():
page.append(f'<h2>{title} data</h2><p><a href="{relative}">Download CSV</a></p><div class="table"><table>')
with path.open(newline='') as stream:
for index, row in enumerate(csv.reader(stream)):
tag = 'th' if index == 0 else 'td'
page.append('<tr>' + ''.join(f'<{tag}>{escape(value)}</{tag}>' for value in row) + '</tr>')
page.append('</table></div>')
page.append('<h2>Files</h2><ul>')
for path in sorted([*root.glob('*.log'), *root.glob('plots/*')]):
relative = path.relative_to(root).as_posix()
page.append(f'<li><a href="{relative}">{escape(relative)}</a></li>')
page.append('</ul></html>')
(root / 'index.html').write_text('\n'.join(page) + '\n')
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('command', choices=('reproduce', 'smoke', 'plots'))
parser.add_argument('--smoke', action='store_true', help='Plot the smoke results.')
parser.add_argument('--jobs', type=int, default=int(os.environ.get('JOBS', len(os.sched_getaffinity(0)))))
parser.add_argument('--threads-per-job', type=int,
default=int(os.environ.get('THREADS_PER_JOB', len(os.sched_getaffinity(0)))))
args = parser.parse_args()
smoke = args.command == 'smoke' or args.smoke
root = Path('/results') / ('smoke' if smoke else 'full')
os.environ['CARGO_NET_OFFLINE'] = 'true'
if args.command == 'plots':
manifest_path = root / 'run.json'
manifest = json.loads(manifest_path.read_text())
records = plots(root, smoke)
manifest['stages'].update(records)
manifest_path.write_text(json.dumps(manifest, indent=2) + '\n')
show_results(root)
return int(any(record['exit_code'] for record in records.values()))
if root.exists():
parser.error(f'{root} exists. Select a fresh OUTPUT_DIR to prevent stale result reuse.')
jobs = args.jobs
if jobs < 1 or args.threads_per_job < 1:
parser.error('jobs and threads-per-job must be positive')
root.mkdir(parents=True)
shutil.copytree(HERE / 'provenance', root / 'provenance')
# Copy the artifact scripts so each result identifies its orchestration code.
shutil.copytree(HERE, root / 'provenance/artifact',
ignore=shutil.ignore_patterns('provenance', '__pycache__'))
manifest = {'started_utc': datetime.now(timezone.utc).isoformat(),
'profile': 'smoke' if smoke else 'full', 'image_id': os.environ.get('ARTIFACT_IMAGE_ID'),
'platform': platform.platform(), 'machine': platform.machine(),
'cpu_count': os.cpu_count(), 'jobs': jobs, 'threads_per_job': args.threads_per_job,
'batch_size': 64, 'fast': False, 'stages': {}}
common = ['--jobs', str(jobs), '--threads-per-job', str(args.threads_per_job),
'--batch-size', '64', '--no-fast']
subset = ['--models', 'vgg8', '--archs', 'arch-a', '--mode', 'latency', '--pipeline', '1'] if smoke else []
commands = {
'experiments': [str(PYTHON), str(COMPARE / 'run_pimcomp_models.py'),
'--out-dir', str(root / 'comparison'),
'--with-ablation', '--ablation-out-dir', str(root / 'ablation'),
'--common-dir', str(root / 'references'), *common, *subset],
}
manifest_path = root / 'run.json'
manifest_path.write_text(json.dumps(manifest, indent=2) + '\n')
for name, command in commands.items():
manifest['stages'][name] = stage(name, command, root)
manifest_path.write_text(json.dumps(manifest, indent=2) + '\n')
manifest['stages'].update(plots(root, smoke))
manifest['finished_utc'] = datetime.now(timezone.utc).isoformat()
manifest_path.write_text(json.dumps(manifest, indent=2) + '\n')
show_results(root)
return int(any(record['exit_code'] for record in manifest['stages'].values()))
if __name__ == '__main__':
raise SystemExit(main())
@@ -0,0 +1,153 @@
"""Run experiment subprocesses as their dependencies complete."""
from __future__ import annotations
import argparse
from collections import deque
from dataclasses import dataclass
import json
import os
from pathlib import Path
import signal
import subprocess
import time
from typing import Callable
def available_processors() -> int:
if hasattr(os, "sched_getaffinity"):
return len(os.sched_getaffinity(0))
return os.cpu_count() or 1
def positive_integer(value: str) -> int:
number = int(value)
if number < 1:
raise argparse.ArgumentTypeError("must be positive")
return number
def thread_environment(threads: int) -> dict[str, str]:
return {**os.environ, "OMP_NUM_THREADS": str(threads),
"OMP_THREAD_LIMIT": str(threads), "OMP_DYNAMIC": "FALSE",
"OMP_WAIT_POLICY": "PASSIVE", "OPENBLAS_NUM_THREADS": str(threads),
"MKL_NUM_THREADS": str(threads), "RAYON_NUM_THREADS": str(threads),
"CMAKE_BUILD_PARALLEL_LEVEL": str(threads), "CARGO_BUILD_JOBS": str(threads)}
@dataclass(frozen=True)
class ExperimentTask:
key: str
command: Callable[[], list[str]]
dependencies: tuple[str, ...]
log_path: Path
@dataclass(frozen=True)
class TaskResult:
returncode: int
error: str = ""
def stop_processes(processes: list[subprocess.Popen]) -> None:
for process in processes:
try:
os.killpg(process.pid, signal.SIGTERM)
except ProcessLookupError:
pass
deadline = time.monotonic() + 5
for process in processes:
try:
process.wait(timeout=max(0, deadline - time.monotonic()))
except subprocess.TimeoutExpired:
try:
os.killpg(process.pid, signal.SIGKILL)
except ProcessLookupError:
pass
process.wait()
def run_tasks(tasks: list[ExperimentTask], jobs: int, threads: int,
cwd: Path) -> dict[str, TaskResult]:
if jobs < 1 or threads < 1:
raise ValueError("jobs and threads-per-job must be positive")
by_key = {task.key: task for task in tasks}
if len(by_key) != len(tasks):
raise ValueError("duplicate experiment task")
waiting = {task.key: set(task.dependencies) for task in tasks}
children = {key: [] for key in by_key}
for task in tasks:
for dependency in task.dependencies:
if dependency not in by_key:
raise ValueError(f"unknown dependency: {dependency}")
children[dependency].append(task.key)
ready = deque(key for key in by_key if not waiting[key])
degrees = {key: len(dependencies) for key, dependencies in waiting.items()}
pending = deque(ready)
visited = 0
while pending:
visited += 1
for child in children[pending.popleft()]:
degrees[child] -= 1
if not degrees[child]:
pending.append(child)
if visited != len(tasks):
raise ValueError("experiment dependency cycle")
active = {}
results = {}
records = {}
environment = thread_environment(threads)
def complete(key: str, result: TaskResult) -> None:
results[key] = result
task = by_key[key]
record = records.get(key, {"dependencies": task.dependencies, "threads_per_job": threads})
record.update(returncode=result.returncode, error=result.error)
task.log_path.parent.mkdir(parents=True, exist_ok=True)
task.log_path.with_suffix(".execution.json").write_text(json.dumps(record, indent=2) + "\n")
print(f"[{'Failed' if result.returncode else 'Completed'}] {key}"
f"{': ' + result.error if result.error else ''}", flush=True)
for child in children[key]:
waiting[child].remove(key)
if not waiting[child]:
ready.append(child)
try:
while ready or active:
while ready and len(active) < jobs:
key = ready.popleft()
task = by_key[key]
failed = [dep for dep in task.dependencies if results[dep].returncode]
if failed:
complete(key, TaskResult(1, "prerequisite failed: " + ", ".join(failed)))
continue
log = None
try:
command = task.command()
task.log_path.parent.mkdir(parents=True, exist_ok=True)
log = task.log_path.open("wb")
metadata = {"command": command, "threads_per_job": threads,
"dependencies": task.dependencies}
records[key] = metadata
task.log_path.with_suffix(".execution.json").write_text(json.dumps(metadata, indent=2) + "\n")
process = subprocess.Popen(command, cwd=cwd, env=environment,
stdout=log, stderr=subprocess.STDOUT, start_new_session=True)
except (OSError, ValueError, RuntimeError) as exc:
if log is not None:
log.close()
complete(key, TaskResult(1, str(exc)))
continue
active[key] = process, log
print(f"[Started] {key}; log: {task.log_path}", flush=True)
completed = [(key, process.returncode) for key, (process, _) in active.items()
if process.poll() is not None]
for key, returncode in completed:
_, log = active.pop(key)
log.close()
complete(key, TaskResult(returncode))
if active and not completed:
time.sleep(.05)
return results
finally:
stop_processes([process for process, _ in active.values()])
for _, log in active.values():
log.close()
@@ -0,0 +1,187 @@
from __future__ import annotations
import argparse
import json
from pathlib import Path
import sys
import pytest
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tools/pim/pimcomp/compare"))
from raptor_validation import experiment_scheduler as scheduler
import run_pimcomp_models as suite
import compare_raptor_pimcomp_model as comparator
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tools/pim/ablation"))
import run_ablation as ablation
def task(root, key, code, dependencies=()):
command = [sys.executable, "-c", code]
return scheduler.ExperimentTask(key, lambda: command, dependencies, root / f"{key}.log")
def test_dependency_starts_before_unrelated_job_finishes(tmp_path):
# The slow job cannot finish until the fast job's dependent has run.
marker = tmp_path / "released"
slow = ("import pathlib,time; deadline=time.monotonic()+5; "
f"p=pathlib.Path({str(marker)!r}); "
"exec('while not p.exists() and time.monotonic()<deadline: time.sleep(.01)'); "
"assert p.exists(), 'dependent was blocked by an unrelated job'")
tasks = [task(tmp_path, "fast", "pass"), task(tmp_path, "slow", slow),
task(tmp_path, "dependent", f"from pathlib import Path; Path({str(marker)!r}).touch()", ("fast",))]
results = scheduler.run_tasks(tasks, 2, 1, tmp_path)
assert all(result.returncode == 0 for result in results.values())
def test_failure_skips_dependents_but_runs_other_jobs(tmp_path):
marker = tmp_path / "must-not-run"
tasks = [task(tmp_path, "failed", "raise SystemExit(3)"),
task(tmp_path, "dependent", f"from pathlib import Path; Path({str(marker)!r}).touch()", ("failed",)),
task(tmp_path, "independent", "pass")]
results = scheduler.run_tasks(tasks, 2, 1, tmp_path)
assert results["failed"].returncode == 3
assert results["dependent"].returncode != 0
assert results["independent"].returncode == 0
assert not marker.exists()
def test_thread_environment_and_execution_record(tmp_path):
names = ["OMP_NUM_THREADS", "OMP_THREAD_LIMIT", "RAYON_NUM_THREADS",
"CMAKE_BUILD_PARALLEL_LEVEL", "CARGO_BUILD_JOBS"]
job = task(tmp_path, "environment", f"import os,json; print(json.dumps({{k:os.environ[k] for k in {names!r}}}))")
scheduler.run_tasks([job], 1, 3, tmp_path)
assert set(json.loads(job.log_path.read_text()).values()) == {"3"}
record = json.loads(job.log_path.with_suffix(".execution.json").read_text())
assert record["threads_per_job"] == 3
assert scheduler.thread_environment(3)["OMP_WAIT_POLICY"] == "PASSIVE"
def test_concurrent_process_limit(tmp_path):
jobs = []
for index in range(6):
output = tmp_path / f"interval-{index}.json"
code = ("import json,time; from pathlib import Path; start=time.monotonic_ns(); "
f"time.sleep(.05); Path({str(output)!r}).write_text(json.dumps([start,time.monotonic_ns()]))")
jobs.append(task(tmp_path, str(index), code))
results = scheduler.run_tasks(jobs, 2, 1, tmp_path)
assert all(result.returncode == 0 for result in results.values())
events = []
for path in tmp_path.glob("interval-*.json"):
start, finish = json.loads(path.read_text())
events.extend(((start, 1), (finish, -1)))
active = 0
for _, delta in sorted(events):
active += delta
assert active <= 2
assert len(events) == 12 and active == 0
def test_interrupt_terminates_workers(tmp_path, monkeypatch):
processes = []
popen = scheduler.subprocess.Popen
def start(*args, **kwargs):
process = popen(*args, **kwargs)
processes.append(process)
return process
monkeypatch.setattr(scheduler.subprocess, "Popen", start)
sleep = scheduler.time.sleep
interrupted = False
def interrupt_once(seconds):
nonlocal interrupted
if not interrupted:
interrupted = True
raise KeyboardInterrupt()
sleep(seconds)
monkeypatch.setattr(scheduler.time, "sleep", interrupt_once)
with pytest.raises(KeyboardInterrupt):
scheduler.run_tasks([task(tmp_path, "waiting", "import time; time.sleep(30)")], 1, 1, tmp_path)
assert processes and all(process.poll() is not None for process in processes)
def test_invalid_dependency_graph_is_rejected(tmp_path):
with pytest.raises(ValueError, match="unknown dependency"):
scheduler.run_tasks([task(tmp_path, "a", "pass", ("missing",))], 1, 1, tmp_path)
with pytest.raises(ValueError, match="cycle"):
scheduler.run_tasks([task(tmp_path, "a", "pass", ("b",)),
task(tmp_path, "b", "pass", ("a",))], 1, 1, tmp_path)
def test_defaults_follow_processor_count(monkeypatch):
monkeypatch.setattr(suite, "available_processors", lambda: 7)
args = suite.parse_args([])
assert args.jobs == args.threads_per_job == 7
args = suite.parse_args(["--jobs", "2", "--threads-per-job", "3"])
assert (args.jobs, args.threads_per_job) == (2, 3)
with pytest.raises(SystemExit):
suite.parse_args(["--threads-per-job", "0"])
monkeypatch.setattr(ablation, "available_processors", lambda: 7)
args, forwarded = ablation.parse_args([])
assert args.jobs == args.threads_per_job == 7
assert forwarded[forwarded.index("--threads-per-job") + 1] == "7"
def test_combined_plan_reuses_references_and_raptor_baselines(tmp_path, monkeypatch):
monkeypatch.setattr(suite, "validate_pimcomp_source", lambda: None)
args = suite.parse_args(["--out-dir", str(tmp_path / "comparison"),
"--ablation-out-dir", str(tmp_path / "ablation"), "--common-dir", str(tmp_path / "references"),
"--jobs", "2", "--threads-per-job", "2", "--dry-run"])
comparison = suite.make_plan(args)
ablations, _, _ = suite.combined_ablation_plans(args)
tasks = suite.build_tasks([comparison, *ablations])
assert len([job for job in tasks if job.key.startswith("reference:")]) == 5
assert len(tasks) == 5 + 40 + 24 # 16 baseline cases reuse comparison reports.
assert sum(len(plan.aliases) for plan in ablations) == 16
assert len({job.key for job in tasks}) == len(tasks)
for job in tasks:
if job.key.startswith("reference:"):
command = job.command()
assert command[command.index("--jobs") + 1] == "2"
continue
command = job.command()
assert command[command.index("--threads-per-job") + 1] == "2"
if "--reuse-pimcomp-report" in command:
assert len(job.dependencies) == 2
else:
assert len(job.dependencies) == 1
def test_raptor_receives_thread_limit(tmp_path, monkeypatch):
received = []
monkeypatch.setattr(comparator, "compile_with_raptor",
lambda *args, **kwargs: received.extend(kwargs["raptor_extra_args"]) or {})
args = argparse.Namespace(threads_per_job=3, pimcomp_config=tmp_path / "config.json",
raptor_extra_arg=[], raptor_path=tmp_path / "compiler", verbose_raptor_compile=False, timeout_seconds=0)
comparator.compile_raptor_target(tmp_path / "model.onnx", tmp_path / "output",
{"core_count": 1, "crossbar_count": 1, "crossbar_size": 128}, args, [])
assert "-j=3" in received
def test_stale_report_is_not_published_after_missing_execution(tmp_path):
args = suite.parse_args(["--out-dir", str(tmp_path), "--models", "vgg8",
"--archs", "arch-a", "--mode", "latency", "--raptor-only"])
plan = suite.make_plan(args)
report = suite.report_for(plan, plan.specs[0])
report.parent.mkdir(parents=True)
report.write_text(json.dumps({"validation_policy": suite.VALIDATION_POLICY,
"raptor_validation": {"status": "done", "passed": True},
"raptor_performance": {"latency_ms": 1}}))
tasks = suite.build_tasks([plan])
assert suite.finish_plan(plan, {task.key: suite.TaskResult(0) for task in tasks}) == 1
import csv
with (tmp_path / "results_comparison.csv").open() as stream:
row = next(csv.DictReader(stream))
assert row["raptor_latency_ms"] == row["raptor_functional_validation"] == "NA"
def test_reference_workers_do_not_create_nested_thread_teams(tmp_path, monkeypatch):
environments = []
monkeypatch.setattr(comparator, "save_inputs_to_files", lambda *a: ([], None))
monkeypatch.setattr(comparator, "run_logged", lambda *a, **k: environments.append(k["env"]))
comparator.generate_reference_outputs(tmp_path / "runner", tmp_path, tmp_path / "model.onnx", [], [],
argparse.Namespace(timeout_seconds=0), tmp_path / "output")
assert environments[0]["OMP_NUM_THREADS"] == environments[0]["OPENBLAS_NUM_THREADS"] == "1"
+40 -21
View File
@@ -524,23 +524,32 @@ def test_suite_reruns_anchor_and_shares_only_new_reports(tmp_path, monkeypatch,
report.parent.mkdir(parents=True)
report.write_text("{}")
os.utime(report, ns=(1, 1))
batches = []
commands = []
errors = []
def run_jobs(jobs, max_workers, log_dir, offset):
assert max_workers == 1
batches.append([command for _, command, _ in jobs])
for _, command, _ in jobs:
output = Path(command[command.index("--out-dir") + 1]) / "pimcomp/comparison_report.json"
if fresh or len(batches) > 1:
output.parent.mkdir(parents=True, exist_ok=True)
output.write_text('{"new": true}')
return [], offset + len(jobs)
def run_jobs(tasks, jobs, threads, cwd):
assert jobs == 1
results = {}
for task in tasks:
try:
command = task.command()
except ValueError as exc:
errors.append(str(exc))
results[task.key] = suite.TaskResult(1, str(exc))
continue
if "--prepare-common" not in command:
commands.append(command)
output = Path(command[command.index("--out-dir") + 1]) / "pimcomp/comparison_report.json"
if fresh or only == "raptor":
output.parent.mkdir(parents=True, exist_ok=True)
output.write_text('{"new": true}')
results[task.key] = suite.TaskResult(0)
return results
summary = tmp_path / "results.csv"
summary.write_text("summary\n")
monkeypatch.setattr(suite, "validate_pimcomp_source", lambda: None)
monkeypatch.setattr(suite, "run", lambda *a, **k: 0)
monkeypatch.setattr(suite, "run_comparison_jobs", run_jobs)
monkeypatch.setattr(suite, "run_tasks", run_jobs)
monkeypatch.setattr(suite, "write_results_csv", lambda *a: summary)
monkeypatch.setattr(suite, "comparison_passed", lambda report: True)
monkeypatch.setattr(sys, "argv", [
@@ -548,15 +557,19 @@ def test_suite_reruns_anchor_and_shares_only_new_reports(tmp_path, monkeypatch,
"--archs", "arch-a", "--mode", "throughput", "--jobs", "1",
*(["--only", only] if only else []),
])
assert suite.main() == 0
assert suite.main() == (1 if only != "raptor" and not fresh else 0)
if only == "raptor":
assert len(batches[0]) == 3
assert all("--reuse-pimcomp-report" in command for command in batches[0])
assert len(commands) == 3
assert all("--reuse-pimcomp-report" in command for command in commands)
else:
assert len(batches[0]) == 1
assert "--reuse-pimcomp-report" not in batches[0][0]
assert all(("--reuse-pimcomp-report" in command) == fresh for command in batches[1])
assert ("--reuse-raptor-report" in batches[0][0]) == (only == "pimcomp")
assert "--reuse-pimcomp-report" not in commands[0]
assert ("--reuse-raptor-report" in commands[0]) == (only == "pimcomp")
if fresh:
assert len(commands) == 3
assert all("--reuse-pimcomp-report" in command for command in commands[1:])
else:
assert len(commands) == 1 # Missing reports must not cause duplicate Pimcomp work.
assert len(errors) == 2
def test_missing_pimcomp_report_is_not_reused(tmp_path):
@@ -658,7 +671,7 @@ def test_ablation_cli_selects_canonical_or_custom_output(tmp_path, monkeypatch,
canonical = tmp_path / "scripts/results_ablation.csv"
monkeypatch.setattr(ablation, "DEFAULT_OUT_DIR", artifact_root)
monkeypatch.setattr(ablation, "DEFAULT_RESULTS_PATH", canonical)
monkeypatch.setattr(ablation.subprocess, "Popen", lambda *a, **k: argparse.Namespace(wait=lambda: 0))
monkeypatch.setattr(suite, "run_plans", lambda plans: 0)
calls = []
monkeypatch.setattr(ablation, "aggregate_results", lambda root, *a, **k: (calls.append((root, k["output"])) or None, []))
command = ["--variant", "baseline", "--jobs", "1"]
@@ -863,7 +876,13 @@ def test_ablation_runner_distinguishes_invalid_outputs_from_failed_execution(tmp
csv = tmp_path / "results.csv"
csv.write_text("summary\n")
monkeypatch.setattr(suite, "run", lambda *a, **k: 0)
monkeypatch.setattr(suite, "run_comparison_jobs", lambda *a: (["arch-a/vgg8/throughput/pipeline4"] if process_failed else [], 1))
def complete_tasks(tasks, *args):
previous = output.stat().st_mtime_ns
output.write_text(json.dumps(report))
os.utime(output, ns=(previous + 1000000, previous + 1000000))
return {task.key: suite.TaskResult(int(process_failed)) for task in tasks}
monkeypatch.setattr(suite, "run_tasks", complete_tasks)
monkeypatch.setattr(suite, "write_results_csv", lambda *a: csv)
monkeypatch.setattr(sys, "argv", ["run_pimcomp_models.py", "--out-dir", str(tmp_path),
"--models", "vgg8", "--archs", "arch-a", "--mode", "throughput", "--pipeline", "4",
+4 -1
View File
@@ -9,6 +9,9 @@ Run the synchronization/Spatial-planning ablation from the repository root:
Defaults: `vgg8 resnet18 resnet34 googlenet`, `arch-a arch-b`, latency
(pipeline 1), and throughput (pipeline 4). Select `yolo11n` or `arch-c`
explicitly when needed. Only Raptor runs; Pimcomp validation is `SKIPPED`.
Ready models, architectures, modes, and variants run concurrently in one dependency queue.
References are prepared once per model. `--jobs` and `--threads-per-job` both default
to the available logical processor count; override them to limit CPU contention.
| Variant | Raptor options |
|---|---|
@@ -34,7 +37,7 @@ suite pipelines for that mode. Pipeline-only selection uses its matching mode.
`--pipeline auto` requests automatic throughput depth selection; positive numeric
depths retain their exact-stage meaning.
`--only` is rejected because
the wrapper always runs Raptor only. Ctrl+C stops the active variant's workers.
the wrapper always runs Raptor only. Ctrl+C stops all active experiment workers.
Artifacts use `<out-dir>/<model>/artifacts/<arch>/<mode>[/pipelineN]/ablation/<variant>/`;
reference inputs, outputs, and runners are shared under `artifacts/common/`.
+21 -41
View File
@@ -6,10 +6,7 @@ from __future__ import annotations
import argparse
import csv
import math
import os
import shlex
import signal
import subprocess
import sys
from dataclasses import dataclass
from pathlib import Path
@@ -26,6 +23,7 @@ from raptor_validation.pimcomp_models import (
)
from raptor_validation.artifacts import remove_lock_files
from raptor_validation.raptor import parse_pipeline_depth, pipeline_sort_key
from raptor_validation.experiment_scheduler import available_processors, positive_integer
@dataclass(frozen=True)
@@ -79,9 +77,14 @@ def parse_args(argv: list[str] | None = None) -> tuple[argparse.Namespace, list[
help="Run only this variant; baseline is added when needed. Repeat as needed.",
)
parser.add_argument("--dry-run", action="store_true", help="Print runner commands without modifying files.")
parser.add_argument("--jobs", "-j", type=positive_integer, default=available_processors(),
help="Maximum concurrent cases (default: available logical processors).")
parser.add_argument("--threads-per-job", type=positive_integer, default=available_processors(),
help="Threads per case (default: available logical processors).")
args, forwarded = parser.parse_known_args(argv)
if any(option == "--only" or option.startswith("--only=") for option in forwarded):
parser.error("--only is not supported; the ablation wrapper always runs Raptor only")
forwarded.extend(("--jobs", str(args.jobs), "--threads-per-job", str(args.threads_per_job)))
return args, forwarded
@@ -156,23 +159,6 @@ def runner_command(
return command
def terminate_process_group(process: subprocess.Popen[bytes]) -> None:
if process.poll() is not None:
return
try:
os.killpg(process.pid, signal.SIGTERM)
except ProcessLookupError:
return
try:
process.wait(timeout=5)
except subprocess.TimeoutExpired:
try:
os.killpg(process.pid, signal.SIGKILL)
except ProcessLookupError:
return
process.wait()
def percentage_difference(
value: str | None, reference: str | None, *, lower_is_better: bool = False
) -> str:
@@ -313,33 +299,27 @@ def main(argv: list[str] | None = None) -> int:
)
return 0
out_dir.mkdir(parents=True, exist_ok=True)
failed = []
current_process = None
sys.path.insert(0, str(RUNNER.parent))
import run_pimcomp_models as suite
plans = []
try:
for variant in variants:
for case_label, case_forwarded in case_sets:
for _, case_forwarded in case_sets:
if variant.name == "no-sync" and case_key(case_forwarded)[0] != "throughput":
continue
command = runner_command(
out_dir, variant, args.models, case_forwarded, out_dir, False
)
print(f"[{variant.name}/{case_label}] {shlex.join(command)}")
current_process = subprocess.Popen(
command,
cwd=REPO,
start_new_session=True,
)
returncode = current_process.wait()
current_process = None
if returncode:
failed.append(f"{variant.name}/{case_label}: runner exited with {returncode}")
command = runner_command(out_dir, variant, args.models, case_forwarded, out_dir, False)
selected = suite.parse_args(command[2:])
if selected.clean:
print(f"Removed {suite.clean_artifacts(out_dir, args.models, out_dir)} artifact path(s).")
return 0
plans.append(suite.make_plan(selected))
failed = ["experiment jobs failed"] if suite.run_plans(plans) else []
except KeyboardInterrupt:
if current_process is not None:
terminate_process_group(current_process)
remove_lock_files(out_dir)
print("Interrupted; terminated the active ablation job.", file=sys.stderr)
print("Interrupted; terminated ablation workers.", file=sys.stderr)
return 130
except ValueError as exc:
print(str(exc), file=sys.stderr)
return 2
remove_lock_files(out_dir)
selected_cases = frozenset(case_key(arguments) for _, arguments in case_sets)
+17 -2
View File
@@ -11,7 +11,12 @@ built release Raptor, Pimcomp, and `pimsim-nn`.
```
Defaults: all five models, Arch-A/B, latency pipeline 1, throughput pipelines
2/4/8, four workers, and a functional throughput batch of 64.
2/4/8 and a functional throughput batch of 64.
Both `--jobs` and `--threads-per-job` default to the available logical processor count.
Their limits multiply: 40 jobs with 40 threads can create 1,600 threads.
Use `--jobs 8 --threads-per-job 4` to reduce contention.
The thread limit controls OpenMP, Raptor's compilation thread pool (`-j`), BLAS, and Rayon.
Reference batches use that many workers, with one thread per reference process.
Use `--help` for the complete option reference. Numeric depths greater than one
require exactly that many active compute stages. `--pipeline auto` requests
throughput depth selection. Reports include `raptor_selected_pipeline`; the
@@ -26,6 +31,8 @@ requested depth remains in the existing pipeline field.
| `--only {raptor,pimcomp}` | Rerun one compiler and reuse the other's existing report. |
| `--raptor-only` | Run Raptor without invoking Pimcomp. |
| `--batch-size N`, `--jobs N` | Set functional throughput batch size and worker count. |
| `--threads-per-job N` | Set the thread limit inside each job. |
| `--with-ablation`, `--ablation-out-dir PATH` | Schedule both suites together; optionally select a separate ablation root. |
| `--pimsim-time-ms N` | Throughput convergence deadline; default: 1000 ms. |
| `--no-fast` | Use full-duration throughput simulation for authoritative experiments. |
| `--timeout-seconds N` | Per-stage timeout; default: 0 (unlimited). |
@@ -45,11 +52,19 @@ Artifacts use
under `artifacts/common/`
and `comparison_report.{md,json}` under each comparison's `pimcomp/`.
YOLO and GoogLeNet use `artifacts/common-full-model/` to avoid reusing historical Softmax-free references.
References are prepared once per model. Each case starts when its own prerequisites finish.
Pimcomp runs once per model/architecture/mode; later pipelines reuse its newly
written report, including failures without simulation artifacts. Missing or
unchanged anchor reports trigger fresh attempts. The runner continues after failures and exits nonzero if a
unchanged prerequisite reports skip dependent cases without repeating Pimcomp work.
Skipped cases retain unavailable metrics rather than old measurements.
The runner continues after failures and exits nonzero if a
compiler, simulation, or functional comparison fails.
Combined runs share references with ablation and reuse matching Raptor baseline reports.
`ablation/baseline/source_reports.json` records those report locations.
Only the parent process writes aggregate CSVs. Per-case logs and execution settings remain beside each case.
Thread limits are recorded in reports and checked during report reuse.
If fast-mode convergence fails, the reports and CSV retain valid full-duration
fallback throughput and output interval values. This is not a converged steady-state result.
Per-case JSON records `throughput_measurement: full_duration_fallback` and
@@ -46,6 +46,7 @@ from raptor_validation.onnx_utils import (
)
from raptor_validation.raptor import compile_with_raptor, read_selected_pipeline_depth
from raptor_validation.output_comparison import VALIDATION_POLICY, compare_output_arrays
from raptor_validation.experiment_scheduler import available_processors, positive_integer, thread_environment
from raptor_validation.pimsim_nn import (
export_raptor_pimsim_artifact,
parse_pimsim_nn_metrics,
@@ -156,6 +157,7 @@ def run_logged(
steps: list[StepRecord],
stage: str | None = None,
print_header: bool = True,
env: dict[str, str] | None = None,
) -> str:
if print_header:
print_step(name, cmd, cwd, stage)
@@ -172,6 +174,7 @@ def run_logged(
stderr=subprocess.STDOUT,
text=True,
timeout=None if timeout_sec == 0 else timeout_sec,
env=env,
)
except subprocess.TimeoutExpired as exc:
duration = time.perf_counter() - start
@@ -364,7 +367,8 @@ def compile_reference(
run_logged(
"Compile reference ONNX IR",
[str(args.raptor_path), str(model_path), "-o", str(onnx_ir_base), "--EmitONNXIR",
"--mlir-elide-elementsattrs-if-larger=16", "--enable-conv-opt-pass=false"],
"--mlir-elide-elementsattrs-if-larger=16", "--enable-conv-opt-pass=false",
f"-j={args.threads_per_job}"],
cwd=REPO,
timeout_sec=args.timeout_seconds,
steps=steps,
@@ -372,7 +376,7 @@ def compile_reference(
)
run_logged(
"Compile reference native",
[str(args.raptor_path), "-O3", str(model_path), "-o", str(runner_base)],
[str(args.raptor_path), "-O3", str(model_path), "-o", str(runner_base), f"-j={args.threads_per_job}"],
cwd=REPO,
timeout_sec=args.timeout_seconds,
steps=steps,
@@ -400,7 +404,7 @@ def compile_reference(
)
run_logged(
"Build runner",
["cmake", "--build", ".", "-j"],
["cmake", "--build", ".", "--parallel", str(args.threads_per_job)],
cwd=build_dir,
timeout_sec=args.timeout_seconds,
steps=steps,
@@ -446,6 +450,7 @@ def generate_reference_outputs(
timeout_sec=args.timeout_seconds,
steps=steps,
print_header=print_header,
env=thread_environment(1),
)
return reference_dir
@@ -576,6 +581,7 @@ def compile_raptor_target(
f"--core-count={hardware['core_count']}",
f"--pim-target-config={args.pimcomp_config}",
"--pim-emit-json",
f"-j={args.threads_per_job}",
*args.raptor_extra_arg,
]
print_step("Compile Raptor", cmd, REPO, "Compile Pim")
@@ -584,6 +590,7 @@ def compile_raptor_target(
raptor_extra_args = [
f"--pim-target-config={args.pimcomp_config}",
"--pim-emit-json",
f"-j={args.threads_per_job}",
*args.raptor_extra_arg,
]
try:
@@ -1202,6 +1209,7 @@ def experiment_context(args, model, functional_model, hardware, side):
"oracle": args.pimsim_nn_build_dir / "ChipTest",
"comparison": Path(__file__),
"validation": VALIDATION_DIR / "raptor_validation/output_comparison.py",
"thread_policy": VALIDATION_DIR / "raptor_validation/experiment_scheduler.py",
"functional_simulator": args.pim_simulator_dir / "target/release/pim-simulator"}
if side == "raptor":
files["compiler"] = args.raptor_path
@@ -1216,6 +1224,7 @@ def experiment_context(args, model, functional_model, hardware, side):
settings.update({name: getattr(args, name) for name in (
("raptor_extra_arg",) if side == "raptor" else
("pimcomp_pipeline", "pimcomp_replication", "pimcomp_model_name"))})
settings["threads_per_job"] = getattr(args, "threads_per_job", available_processors())
return {"version": 2, "validation_policy": VALIDATION_POLICY,
"files": fingerprint_files(files), "hardware": hardware, "settings": settings}
@@ -1569,9 +1578,9 @@ def main():
parser.add_argument(
"-j",
"--jobs",
type=int,
default=4,
help="Maximum parallel native reference runner processes (default: 4).",
type=positive_integer,
default=available_processors(),
help="Maximum parallel native reference runner processes (default: logical processors).",
)
parser.add_argument("--pimcomp-pipeline", choices=["element", "batch"])
parser.add_argument("--pimcomp-model-name", help="Use a Pimcomp built-in model name such as vgg16.")
@@ -1613,7 +1622,10 @@ def main():
action="store_true",
help="Return a non-zero status if a stage or semantic validation fails.",
)
parser.add_argument("--threads-per-job", type=positive_integer, default=available_processors(),
help="Threads for OpenMP, MLIR compilation, and simulation (default: logical processors).")
args = parser.parse_args()
os.environ.update(thread_environment(args.threads_per_job))
if args.raptor_only and any(
option is not None
for option in (
@@ -2152,6 +2164,7 @@ def main():
"hardware": hardware,
"pimsim_mode": args.pimsim_mode,
"pimsim_time_ms": args.pimsim_time_ms,
"threads_per_job": args.threads_per_job,
"pimcomp_pipeline": args.pimcomp_pipeline,
"pimcomp_replication": args.pimcomp_replication,
"pimcomp_model_source": "not_run" if args.raptor_only else "original_onnx",
@@ -5,16 +5,12 @@ import argparse
import csv
import json
import math
import os
import shlex
import shutil
import subprocess
import sys
from concurrent.futures import ProcessPoolExecutor, as_completed
from contextlib import nullcontext, redirect_stderr, redirect_stdout
from dataclasses import dataclass
from pathlib import Path
from tempfile import TemporaryDirectory
from colorama import Fore, Style
@@ -31,7 +27,9 @@ from raptor_validation.artifacts import artifacts_dir, remove_lock_files
from raptor_validation.pimsim_nn import parse_pimsim_nn_metrics
from raptor_validation.output_comparison import VALIDATION_POLICY
from raptor_validation.raptor import parse_pipeline_depth, pipeline_sort_key
from raptor_validation.validate_one import STAGE_COLORS
from raptor_validation.experiment_scheduler import (
ExperimentTask, TaskResult, available_processors, positive_integer, run_tasks,
)
PIMCOMP_SOURCE = REPO / "third_party/PIMCOMP-NN"
PIMCOMP_CONFIGS = REPO / "validation/pimsim_configs/pimcomp"
@@ -130,6 +128,8 @@ def write_results_csv(
models: list[str],
comparisons: tuple[tuple[str, int, str], ...] = COMPARISONS,
ablation_variant: str | None = None,
report_paths: dict[Path, Path | None] | None = None,
raptor_only: bool = False,
) -> Path:
output = DEFAULT_RESULTS_PATH if root is None else root / RESULTS_FILENAME
if ablation_variant is not None:
@@ -179,6 +179,9 @@ def write_results_csv(
report_path = result_dir(
root, name, arch, comparison_mode, pipeline, ablation_variant
) / "pimcomp/comparison_report.json"
if report_paths is not None:
directory = result_dir(root, name, arch, comparison_mode, pipeline, ablation_variant)
report_path = report_paths.get(directory, report_path)
row = {
"model": name,
"arch": arch,
@@ -198,12 +201,12 @@ def write_results_csv(
"better_compiler": "NA",
"speedup": "NA",
}
if report_path.exists():
if report_path is not None and report_path.exists():
report = json.loads(report_path.read_text(encoding="utf-8"))
if report.get("validation_policy") != VALIDATION_POLICY:
raise ValueError(f"Report uses a different validation policy: {report_path}. Regenerate this comparison.")
raptor_values = performance_values(report.get("raptor_performance") or {})
pimcomp_values = performance_values(report.get("pimcomp_performance") or {})
pimcomp_values = performance_values({} if raptor_only else report.get("pimcomp_performance") or {})
raptor_metric = raptor_values["throughput"] if comparison_mode == "throughput" else raptor_values["latency"]
pimcomp_metric = pimcomp_values["throughput"] if comparison_mode == "throughput" else pimcomp_values["latency"]
row.update(
@@ -211,7 +214,7 @@ def write_results_csv(
report.get("raptor_validation")
),
pimcomp_functional_validation=functional_validation_status(
report.get("pimcomp_validation")
{"status": "skipped"} if raptor_only else report.get("pimcomp_validation")
),
raptor_throughput_samples_s=format_value(raptor_values["throughput"]),
pimcomp_throughput_samples_s=format_value(pimcomp_values["throughput"]),
@@ -322,17 +325,6 @@ def format_value(value: float | None) -> str:
return f"{value:.6g}"
def print_stage(title: str, color: str) -> None:
print(Style.BRIGHT + color + f"[{title}]" + Style.RESET_ALL, flush=True)
def print_completed(label: str, output: str = "") -> None:
print(Style.BRIGHT + Fore.CYAN + f"[Completed {label}]" + Style.RESET_ALL)
if output:
print(output, end="" if output.endswith("\n") else "\n")
print("=" * 72, flush=True)
def run(command: list[str], *, dry_run: bool, check: bool = True) -> int:
print(f" cwd: {REPO}", flush=True)
print(f" $ {shlex.join(command)}", flush=True)
@@ -341,29 +333,6 @@ def run(command: list[str], *, dry_run: bool, check: bool = True) -> int:
return subprocess.run(command, cwd=REPO, check=check).returncode
def run_comparison_job(job: tuple[str, list[str], Path | None]) -> tuple[str, int, str | None]:
label, command, log_path = job
sys.stdout.flush()
sys.stderr.flush()
if log_path is None:
return label, run(command, dry_run=False, check=False), None
saved_stdout = os.dup(1)
saved_stderr = os.dup(2)
try:
with open(log_path, "w", encoding="utf-8", buffering=1) as log:
os.dup2(log.fileno(), 1)
os.dup2(log.fileno(), 2)
with redirect_stdout(log), redirect_stderr(log):
returncode = run(command, dry_run=False, check=False)
finally:
os.dup2(saved_stdout, 1)
os.dup2(saved_stderr, 2)
os.close(saved_stdout)
os.close(saved_stderr)
return label, returncode, log_path.read_text(encoding="utf-8", errors="replace")
def validate_pimcomp_source() -> None:
header = PIMCOMP_SOURCE / "backend/GeneticAlgorithm.h"
source = header.read_text(encoding="utf-8")
@@ -469,7 +438,7 @@ def comparison_command_for(
reuse_pimcomp_report = report
elif reuse_shared_pimcomp:
reuse_pimcomp_report = spec.shared_pimcomp_dir.parent / "comparison_report.json"
return comparison_command(
command = comparison_command(
spec.model,
spec.functional_model,
spec.output_dir,
@@ -491,6 +460,7 @@ def comparison_command_for(
),
reuse_pimcomp_report=reuse_pimcomp_report,
)
return command + ["--jobs", "1", "--threads-per-job", str(args.threads_per_job)]
def pimcomp_report_stamp(output_dir: Path) -> tuple[int, int, int] | None:
@@ -506,40 +476,6 @@ def pimcomp_report_ready(output_dir: Path, previous: tuple[int, int, int] | None
return stamp is not None and stamp != previous
def run_comparison_jobs(
comparison_jobs: list[tuple[str, list[str], Path | None]],
max_workers: int,
log_dir: Path | None,
log_offset: int,
) -> tuple[list[str], int]:
if not comparison_jobs:
return [], log_offset
print_directly = min(max_workers, len(comparison_jobs)) == 1
jobs = [
(
label,
command,
None if print_directly or log_dir is None else log_dir / f"{log_offset + index}.log",
)
for index, (label, command, _) in enumerate(comparison_jobs)
]
with (nullcontext(None) if print_directly else ProcessPoolExecutor(max_workers=max_workers)) as executor:
completed = (
map(run_comparison_job, jobs)
if print_directly
else (
future.result()
for future in as_completed(executor.submit(run_comparison_job, job) for job in jobs)
)
)
failed = []
for label, returncode, output in completed:
print_completed(label, output or "")
if returncode:
failed.append(label)
return failed, log_offset + len(jobs)
def ensure_throughput_config(arch: str, sim_time_ms: int) -> Path:
source = PIMCOMP_CONFIGS / arch / "throughput_config_1000ms.json"
target = PIMCOMP_CONFIGS / arch / f"throughput_config_{sim_time_ms}ms.json"
@@ -566,7 +502,7 @@ def config_path(arch: str, mode: str, sim_time_ms: int, *, write: bool) -> Path:
return path
def main() -> int:
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Compare supported Pimcomp models with Raptor latency and throughput schedules."
)
@@ -635,9 +571,9 @@ def main() -> int:
parser.add_argument(
"-j",
"--jobs",
type=int,
default=4,
help="Number of comparisons to run in parallel (default: 4).",
type=positive_integer,
default=available_processors(),
help="Maximum concurrent comparisons (default: available logical processors).",
)
parser.add_argument(
"--clean",
@@ -657,7 +593,11 @@ def main() -> int:
dest="raptor_extra_args",
help="Additional argument to pass to Raptor; repeat as needed.",
)
args = parser.parse_args()
parser.add_argument("--threads-per-job", type=positive_integer, default=available_processors(),
help="Threads per compiler/simulator job (default: available logical processors).")
parser.add_argument("--with-ablation", action="store_true", help="Run comparison and ablation in one dependency queue.")
parser.add_argument("--ablation-out-dir", type=Path, help="Output root for a combined ablation run.")
args = parser.parse_args(argv)
if args.ablation_variant and Path(args.ablation_variant).name != args.ablation_variant:
parser.error("--ablation-variant must be a single directory name")
@@ -665,25 +605,35 @@ def main() -> int:
if args.only is not None and args.raptor_only:
parser.error("--only cannot be combined with --raptor-only")
return args
@dataclass
class SuitePlan:
args: argparse.Namespace
root: Path | None
common_root: Path | None
comparisons: dict
reference_batch_size: int
specs: list[ComparisonSpec]
aliases: dict[Path, Path]
previous_reports: dict[Path, tuple[int, int, int] | None]
def make_plan(args: argparse.Namespace) -> SuitePlan:
out_dir = args.out_dir.resolve() if args.out_dir is not None else None
common_root = args.common_dir.resolve() if args.common_dir is not None else None
args.archs = list(dict.fromkeys(args.archs))
args.models = list(dict.fromkeys(args.models))
args.mode = list(dict.fromkeys(args.mode))
if args.clean:
print(
f"Removed {clean_artifacts(out_dir, args.models, common_root)} "
"comparison artifact path(s)."
)
return 0
if args.jobs < 1:
parser.error("--jobs must be at least 1")
raise ValueError("--jobs must be at least 1")
if args.pimsim_time_ms <= 0:
parser.error("--pimsim-time-ms must be positive")
raise ValueError("--pimsim-time-ms must be positive")
if args.batch_size <= 0:
parser.error("--batch-size must be positive")
raise ValueError("--batch-size must be positive")
if args.timeout_seconds < 0:
parser.error("--timeout-seconds must be non-negative")
raise ValueError("--timeout-seconds must be non-negative")
comparisons_by_arch: dict[str, tuple[tuple[str, int | str, str], ...]] = {}
configs_by_arch: dict[str, dict[str, Path]] = {}
for arch in args.archs:
@@ -697,7 +647,7 @@ def main() -> int:
and (args.pipeline is None or comparison[1] == args.pipeline)
)
if not comparisons:
parser.error("--pipeline does not belong to the selected --mode")
raise ValueError("--pipeline does not belong to the selected --mode")
configs = {
mode: config_path(arch, mode, args.pimsim_time_ms, write=not args.dry_run)
for mode, _, _ in comparisons
@@ -716,7 +666,7 @@ def main() -> int:
if not MODELS[name].exists()
]
if missing:
parser.error(f"missing model(s): {', '.join(missing)}")
raise ValueError(f"missing model(s): {', '.join(missing)}")
if args.only is not None:
missing_reuse = []
@@ -730,7 +680,7 @@ def main() -> int:
if not required.exists():
missing_reuse.append(str(required))
if missing_reuse:
parser.error(
raise ValueError(
f"--only {args.only} needs existing comparison artifacts: "
+ ", ".join(missing_reuse)
)
@@ -740,41 +690,6 @@ def main() -> int:
if out_dir is not None and not args.dry_run:
out_dir.mkdir(parents=True, exist_ok=True)
print(Style.BRIGHT + f"Found {len(args.models)} Pimcomp model(s) to compare." + Style.RESET_ALL)
print(f"Architectures: {', '.join(args.archs)}")
print(f"Modes: {', '.join(args.mode)}")
print(f"Throughput Pimsim time: {args.pimsim_time_ms} ms")
print(f"Max parallel jobs: {args.jobs}")
print(f"Shared reference batch: {reference_batch_size}")
print(
f"Comparison jobs: "
f"{sum(len(args.models) * len(comparisons) for comparisons in comparisons_by_arch.values())}"
)
print(f"Artifacts root: {out_dir or SUITE}")
print("=" * 72)
print_stage("Prepare shared artifacts", STAGE_COLORS["Build runner"])
try:
for name in args.models:
try:
run(
prepare_common_command(
MODELS[name],
common_dir(out_dir, name, common_root),
args.timeout_seconds,
reference_batch_size,
args.jobs,
),
dry_run=args.dry_run,
)
finally:
print_completed(f"shared artifacts: {name}")
except KeyboardInterrupt:
remove_lock_files(out_dir or SUITE)
print("Interrupted; cleaned validation lock files.", file=sys.stderr)
return 130
failed = []
comparison_specs: list[ComparisonSpec] = []
shared_pimcomp_by_group: dict[tuple[str, str, str], Path] = {}
for arch in args.archs:
@@ -808,117 +723,171 @@ def main() -> int:
)
)
if args.dry_run:
for spec in comparison_specs:
command = comparison_command_for(
spec,
args,
reuse_shared_pimcomp=not spec.anchor,
)
if run(command, dry_run=True, check=False):
return SuitePlan(args, out_dir, common_root, comparisons_by_arch,
reference_batch_size, comparison_specs, {}, {})
def execution_key(plan: SuitePlan, spec: ComparisonSpec) -> tuple:
args = plan.args
return (spec.model, spec.common_dir, spec.config, spec.mode, spec.pipeline,
tuple(args.raptor_extra_args), args.batch_size, args.no_fast,
args.threads_per_job)
def build_tasks(plans: list[SuitePlan]) -> list[ExperimentTask]:
tasks = []
references = {}
comparison_sources = {execution_key(plan, spec): spec.output_dir
for plan in plans if not plan.args.ablation_variant and plan.args.only is None
and not plan.args.raptor_only for spec in plan.specs}
for plan in plans:
for name in plan.args.models:
common = common_dir(plan.root, name, plan.common_root)
previous = references.get(common)
if previous is None or previous[0] < plan.reference_batch_size:
references[common] = (plan.reference_batch_size, name, plan.args)
for common, (batch_size, name, args) in references.items():
command = prepare_common_command(MODELS[name], common, args.timeout_seconds,
batch_size, args.threads_per_job)
command += ["--threads-per-job", str(args.threads_per_job)]
tasks.append(ExperimentTask("reference:" + str(common), lambda command=command: command,
(), common / "prepare.log"))
for plan in plans:
args = plan.args
for spec in plan.specs:
source = comparison_sources.get(execution_key(plan, spec)) if args.ablation_variant == "baseline" else None
report_root = source if source is not None else spec.output_dir
plan.previous_reports[spec.output_dir] = pimcomp_report_stamp(report_root / "pimcomp/output")
if args.ablation_variant == "baseline" and execution_key(plan, spec) in comparison_sources:
plan.aliases[spec.output_dir] = comparison_sources[execution_key(plan, spec)]
continue
reuse = not (spec.anchor or args.only == "raptor" or args.raptor_only)
dependencies = ["reference:" + str(spec.common_dir)]
previous_report = pimcomp_report_stamp(spec.shared_pimcomp_dir) if reuse else None
if reuse:
dependencies.append(str(spec.shared_pimcomp_dir.parent.parent))
def command(spec=spec, args=args, reuse=reuse, previous=previous_report):
if reuse and not args.dry_run and not pimcomp_report_ready(spec.shared_pimcomp_dir, previous):
raise ValueError(f"Pimcomp prerequisite produced no fresh report: {spec.shared_pimcomp_dir}")
return comparison_command_for(spec, args, reuse_shared_pimcomp=reuse)
tasks.append(ExperimentTask(str(spec.output_dir), command, tuple(dependencies),
spec.output_dir / "comparison.log"))
return tasks
def report_for(plan: SuitePlan, spec: ComparisonSpec) -> Path:
return plan.aliases.get(spec.output_dir, spec.output_dir) / "pimcomp/comparison_report.json"
def finish_plan(plan: SuitePlan, results: dict[str, TaskResult]) -> int:
args = plan.args
failed, diagnostic = [], []
overrides = {}
for spec in plan.specs:
directory = plan.aliases.get(spec.output_dir, spec.output_dir)
result = results.get(str(directory))
fresh = pimcomp_report_ready(directory / "pimcomp/output", plan.previous_reports[spec.output_dir])
overrides[spec.output_dir] = report_for(plan, spec) if result and not result.returncode and fresh else None
for arch in args.archs:
output = write_results_csv(plan.root, arch, args.models, plan.comparisons[arch],
args.ablation_variant, overrides, args.raptor_only)
if plan.aliases:
destination = (plan.root or SUITE) / "ablation" / args.ablation_variant / "source_reports.json"
destination.write_text(json.dumps({str(k): str(v) if v is not None else None
for k, v in overrides.items()}, indent=2) + "\n")
for spec in plan.specs:
result = results.get(str(plan.aliases.get(spec.output_dir, spec.output_dir)))
path = report_for(plan, spec)
report = json.loads(path.read_text()) if overrides[spec.output_dir] is not None else {}
if result is None or result.returncode or overrides[spec.output_dir] is None:
failed.append(spec.label)
elif not comparison_passed(report):
if args.ablation_variant == "no-sync" and completed_synchronization_ablation(report):
diagnostic.append(spec.label)
else:
failed.append(spec.label)
elif comparison_specs:
raptor_only_run = args.only == "raptor" or args.raptor_only
anchor_specs = comparison_specs if raptor_only_run else [
spec for spec in comparison_specs if spec.anchor
]
anchor_jobs = [
(
spec.label,
comparison_command_for(spec, args, reuse_shared_pimcomp=False),
None,
)
for spec in anchor_specs
]
dependent_specs = [] if raptor_only_run else [
spec for spec in comparison_specs if not spec.anchor
]
print_directly = min(args.jobs, len(comparison_specs)) == 1
try:
with (nullcontext(None) if print_directly else TemporaryDirectory(prefix="raptor-pimcomp-")) as log_dir:
previous_reports = {
spec.shared_pimcomp_dir: pimcomp_report_stamp(spec.shared_pimcomp_dir)
for spec in anchor_specs
}
anchor_failed, log_offset = run_comparison_jobs(
anchor_jobs,
args.jobs,
Path(log_dir) if log_dir else None,
0,
)
failed.extend(anchor_failed)
dependent_jobs = [
(
spec.label,
comparison_command_for(
spec,
args,
reuse_shared_pimcomp=pimcomp_report_ready(
spec.shared_pimcomp_dir, previous_reports[spec.shared_pimcomp_dir]
),
),
None,
)
for spec in dependent_specs
]
dependent_failed, _ = run_comparison_jobs(
dependent_jobs,
args.jobs,
Path(log_dir) if log_dir else None,
log_offset,
)
failed.extend(dependent_failed)
except KeyboardInterrupt:
remove_lock_files(out_dir or SUITE)
print("Interrupted; cleaned validation lock files.", file=sys.stderr)
return 130
if args.dry_run:
return 1 if failed else 0
remove_lock_files(out_dir or SUITE)
results_path = None
for arch in args.archs:
results_path = write_results_csv(
out_dir,
arch,
args.models,
comparisons_by_arch[arch],
args.ablation_variant,
)
assert results_path is not None
print_stage(results_path.name, STAGE_COLORS["Compare outputs"])
print(results_path.read_text(encoding="utf-8"), end="")
diagnostic_failures = []
for arch in args.archs:
for name in args.models:
for mode, pipeline, _ in comparisons_by_arch[arch]:
label = f"{arch}/{name}/{mode}/pipeline{pipeline}"
report_path = result_dir(
out_dir, name, arch, mode, pipeline, args.ablation_variant
) / "pimcomp/comparison_report.json"
report = json.loads(report_path.read_text(encoding="utf-8")) if report_path.exists() else {}
if not comparison_passed(report):
if (label not in failed and args.ablation_variant == "no-sync"
and completed_synchronization_ablation(report)):
diagnostic_failures.append(label)
elif label not in failed:
failed.append(label)
print(Style.BRIGHT + Fore.CYAN + "Summary" + Style.RESET_ALL)
total_jobs = sum(len(args.models) * len(comparisons) for comparisons in comparisons_by_arch.values())
print(Style.BRIGHT + f"Passed: {total_jobs - len(failed) - len(diagnostic_failures)}" + Style.RESET_ALL)
print(Style.BRIGHT + f"Failed: {len(failed)}" + Style.RESET_ALL)
if diagnostic_failures:
print("No-sync numerical failures (diagnostic only): " + ", ".join(diagnostic_failures))
print(Style.BRIGHT + f"Results: {results_path}" + Style.RESET_ALL)
print(f"Results: {output}; passed: {len(plan.specs) - len(failed) - len(diagnostic)}; "
f"failed: {len(failed)}; diagnostic no-sync failures: {len(diagnostic)}", flush=True)
if failed:
print(
Fore.RED + f"Failed comparisons: {', '.join(failed)}" + Style.RESET_ALL,
file=sys.stderr,
)
return 1
return 0
print("Failed comparisons: " + ", ".join(failed), file=sys.stderr)
remove_lock_files(plan.root or SUITE)
return int(bool(failed))
def run_plans(plans: list[SuitePlan]) -> int:
tasks = build_tasks(plans)
args = plans[0].args
if any((plan.args.jobs, plan.args.threads_per_job) != (args.jobs, args.threads_per_job) for plan in plans):
raise ValueError("all experiment plans must use the same job and thread limits")
if args.dry_run:
for task in tasks:
run(task.command(), dry_run=True, check=False)
return 0
print(f"Concurrent jobs: {args.jobs}; threads per job: {args.threads_per_job}", flush=True)
results = run_tasks(tasks, args.jobs, args.threads_per_job, REPO)
# Only the parent writes aggregate CSVs.
failures = [finish_plan(plan, results) for plan in plans]
return int(any(failures))
def combined_ablation_plans(args: argparse.Namespace) -> tuple[list[SuitePlan], object, Path]:
sys.path.insert(0, str(REPO / "validation/tools/pim/ablation"))
import run_ablation as ablation
from copy import deepcopy
root = args.ablation_out_dir or args.out_dir or SUITE
models = [name for name in args.models if name in ablation.ABLATION_DEFAULT_MODELS]
if not models:
raise ValueError("combined ablation needs at least one of the four ablation models")
cases = [(mode, depth) for mode, depth, _ in COMPARISONS
if mode in args.mode and (depth in (1, 4) if args.pipeline is None else depth == args.pipeline)]
if args.pipeline is not None and not cases:
cases = [("throughput", args.pipeline)]
plans = []
for variant in ablation.VARIANTS:
for mode, depth in cases:
if variant.name == "no-sync" and mode != "throughput":
continue
selected = deepcopy(args)
selected.out_dir, selected.models = root, models
selected.common_dir = args.common_dir or args.out_dir or SUITE
selected.raptor_only, selected.only = True, None
selected.ablation_variant = variant.name
selected.mode, selected.pipeline = [mode], depth
selected.raptor_extra_args += list(variant.raptor_args)
plans.append(make_plan(selected))
return plans, ablation, root
def main(argv: list[str] | None = None) -> int:
args = parse_args(argv)
if args.clean:
print(f"Removed {clean_artifacts(args.out_dir, args.models, args.common_dir)} comparison artifact path(s).")
return 0
try:
plans = [make_plan(args)]
if args.with_ablation:
if args.only or args.raptor_only or args.ablation_variant:
raise ValueError("--with-ablation requires a full compiler comparison")
extra, ablation, ablation_root = combined_ablation_plans(args)
plans.extend(extra)
result = run_plans(plans)
if args.with_ablation and not args.dry_run:
cases = frozenset((spec.mode, str(spec.pipeline)) for plan in extra for spec in plan.specs)
variants = [variant for variant in ablation.VARIANTS
if any(plan.args.ablation_variant == variant.name for plan in extra)]
_, failures = ablation.aggregate_results(ablation_root, variants, cases,
frozenset(extra[0].args.models),
output=ablation.DEFAULT_RESULTS_PATH if args.out_dir is None and args.ablation_out_dir is None else None)
result = int(bool(result or failures))
return result
except KeyboardInterrupt:
print("Interrupted; terminated experiment workers.", file=sys.stderr)
return 130
except ValueError as exc:
print(str(exc), file=sys.stderr)
return 2
if __name__ == "__main__":