Merge remote-tracking branch 'origin/main'
Validate Operations / validate-operations (push) Has been cancelled
Validate Operations / validate-operations (push) Has been cancelled
This commit is contained in:
@@ -73,6 +73,9 @@ differ, and validation defaults to Arch-A.
|
||||
|
||||
## Documentation
|
||||
|
||||
The [paper artifact](artifact/README.md) builds the pinned compiler and simulators
|
||||
in Docker, runs the comparison and ablation experiments, and generates plots.
|
||||
|
||||
Start with the [documentation index](docs/README.md). It links the
|
||||
[compiler architecture](docs/compiler/architecture.md),
|
||||
[compiler options](docs/compiler/options.md), and
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
*
|
||||
!Dockerfile
|
||||
!pins.env
|
||||
!requirements.lock
|
||||
!models.sha256
|
||||
!build.sh
|
||||
!run.py
|
||||
!plot_ablation.py
|
||||
!source.tar.gz
|
||||
!source.sha256
|
||||
!validation-overlay/
|
||||
!validation-overlay/**
|
||||
!validation-overlay.sha256
|
||||
@@ -0,0 +1,7 @@
|
||||
/source.tar.gz
|
||||
/source.tar.gz.tmp
|
||||
/source.sha256
|
||||
/dist/
|
||||
/results/
|
||||
/validation-overlay/
|
||||
/validation-overlay.sha256
|
||||
@@ -0,0 +1,46 @@
|
||||
FROM python:3.12.12-slim-trixie AS python-deps
|
||||
|
||||
ENV DEBIAN_FRONTEND=noninteractive \
|
||||
CC=clang-19 CXX=clang++-19 \
|
||||
LDFLAGS=-fuse-ld=mold \
|
||||
RUSTFLAGS="-C linker=clang-19 -C link-arg=-fuse-ld=mold" \
|
||||
CARGO_HOME=/opt/cargo RUSTUP_HOME=/opt/rustup \
|
||||
PATH=/opt/raptor/.venv/bin:/opt/cargo/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \
|
||||
OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 MPLBACKEND=Agg \
|
||||
PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 \
|
||||
HOME=/tmp XDG_CACHE_HOME=/tmp/cache MPLCONFIGDIR=/tmp/matplotlib
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
build-essential ca-certificates clang-19 curl git libomp-19-dev \
|
||||
libzstd-dev mold ninja-build pkg-config zlib1g-dev \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /opt/artifact
|
||||
COPY pins.env requirements.lock models.sha256 build.sh ./
|
||||
RUN python -m venv /opt/raptor/.venv \
|
||||
&& /opt/raptor/.venv/bin/python -m pip install --no-cache-dir --only-binary=onnx -r requirements.lock
|
||||
|
||||
FROM python-deps AS artifact
|
||||
ARG BUILD_JOBS
|
||||
RUN . ./pins.env && curl --proto '=https' --tlsv1.2 -fsSL https://sh.rustup.rs -o /tmp/rustup.sh \
|
||||
&& sh /tmp/rustup.sh -y --profile minimal --default-toolchain "$RUST_VERSION" \
|
||||
&& rm /tmp/rustup.sh
|
||||
|
||||
COPY source.tar.gz source.sha256 ./
|
||||
RUN sha256sum -c source.sha256 \
|
||||
&& . ./pins.env \
|
||||
&& gzip -dc source.tar.gz > /tmp/raptor-source.tar \
|
||||
&& test "$(git get-tar-commit-id < /tmp/raptor-source.tar)" = "$RAPTOR_COMMIT" \
|
||||
&& tar -xf /tmp/raptor-source.tar -C /opt/raptor \
|
||||
&& rm source.tar.gz /tmp/raptor-source.tar
|
||||
RUN bash build.sh "${BUILD_JOBS:-$(env -u OMP_NUM_THREADS -u OMP_THREAD_LIMIT nproc)}"
|
||||
COPY validation-overlay/ ./validation-overlay/
|
||||
COPY validation-overlay.sha256 ./
|
||||
RUN sha256sum -c validation-overlay.sha256 \
|
||||
&& cp -R validation-overlay/validation/. /opt/raptor/validation/ \
|
||||
&& cp validation-overlay.sha256 /opt/artifact/provenance/
|
||||
COPY run.py plot_ablation.py ./
|
||||
|
||||
WORKDIR /opt/raptor
|
||||
ENTRYPOINT ["/opt/raptor/.venv/bin/python", "/opt/artifact/run.py"]
|
||||
CMD ["reproduce"]
|
||||
@@ -0,0 +1,32 @@
|
||||
# Raptor paper artifact
|
||||
|
||||
Raptor is fixed to commit `4a1fe710b2f0e4e9c0504f4fd9b4c4dd3df53b7d`.
|
||||
|
||||
Requires Docker on Linux or WSL2. The build downloads dependencies and uses Clang 19 and mold.
|
||||
Install the host dependencies:
|
||||
|
||||
```bash
|
||||
sudo apt install -y docker.io docker-cli docker-buildx git
|
||||
sudo systemctl enable --now docker
|
||||
sudo usermod -aG docker "$USER"
|
||||
newgrp docker
|
||||
```
|
||||
|
||||
Log out and back in, then run from this directory:
|
||||
|
||||
```bash
|
||||
bash artifact.sh build
|
||||
bash artifact.sh reproduce
|
||||
```
|
||||
|
||||
Open `results/full/index.html` to view the plots and result tables.
|
||||
The page also links to the CSV files and logs. Failed numerical validation remains visible in the results.
|
||||
|
||||
For a small VGG8 run, use `bash artifact.sh smoke` and open `results/smoke/index.html`.
|
||||
Build jobs, run jobs, and threads per job default to the available logical processor count.
|
||||
Override with `BUILD_JOBS`, `JOBS`, and `THREADS_PER_JOB`, or run `bash artifact.sh reproduce --jobs 8 --threads-per-job 4`.
|
||||
For another run, set `OUTPUT_DIR` to a new directory.
|
||||
|
||||
To prepare the paper attachment, run `bash artifact/artifact.sh package` from the repository root.
|
||||
The archive is written to `artifact/dist/` and includes the pinned sources and models.
|
||||
Updated scheduling code is included separately; compiler and simulator sources stay pinned.
|
||||
Executable
+61
@@ -0,0 +1,61 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
cd -- "$(dirname -- "${BASH_SOURCE[0]}")"
|
||||
source pins.env
|
||||
image="raptor-artifact:${RAPTOR_COMMIT:0:12}"
|
||||
command=${1:-help}
|
||||
if (($#)); then shift; fi
|
||||
case "$command" in
|
||||
prepare)
|
||||
exec bash prepare.sh
|
||||
;;
|
||||
package)
|
||||
bash prepare.sh
|
||||
mkdir -p dist
|
||||
archive="raptor-artifact-${RAPTOR_COMMIT:0:12}.tar.gz"
|
||||
package_dir=$(mktemp -d)
|
||||
trap 'rm -rf -- "$package_dir"' EXIT
|
||||
mkdir "$package_dir/raptor-artifact"
|
||||
cp \
|
||||
README.md artifact.sh prepare.sh Dockerfile .dockerignore pins.env \
|
||||
requirements.lock models.sha256 build.sh run.py plot_ablation.py source.tar.gz source.sha256 \
|
||||
"$package_dir/raptor-artifact/"
|
||||
cp -R validation-overlay "$package_dir/raptor-artifact/"
|
||||
cp validation-overlay.sha256 "$package_dir/raptor-artifact/"
|
||||
tar -czf "dist/$archive" -C "$package_dir" raptor-artifact
|
||||
(cd dist && sha256sum "$archive" > "$archive.sha256")
|
||||
printf 'Attachment: %s/dist/%s\n' "$PWD" "$archive"
|
||||
;;
|
||||
build)
|
||||
[[ -f source.tar.gz ]] || bash prepare.sh
|
||||
if [[ -f ../validation/raptor_validation/experiment_scheduler.py ]]; then
|
||||
bash prepare.sh --overlay-only
|
||||
fi
|
||||
sha256sum -c source.sha256
|
||||
docker build --build-arg "BUILD_JOBS=${BUILD_JOBS:-$(env -u OMP_NUM_THREADS -u OMP_THREAD_LIMIT nproc)}" -t "$image" .
|
||||
;;
|
||||
reproduce|smoke|plots)
|
||||
output=${OUTPUT_DIR:-"$PWD/results"}
|
||||
mkdir -p "$output"
|
||||
output=$(cd -- "$output" && pwd)
|
||||
image_id=$(docker image inspect --format '{{.Id}}' "$image")
|
||||
status=0
|
||||
docker run --rm --init --network none \
|
||||
--user "$(id -u):$(id -g)" \
|
||||
--env "ARTIFACT_IMAGE_ID=$image_id" \
|
||||
--env "JOBS=${JOBS:-$(env -u OMP_NUM_THREADS -u OMP_THREAD_LIMIT nproc)}" \
|
||||
--env "THREADS_PER_JOB=${THREADS_PER_JOB:-$(env -u OMP_NUM_THREADS -u OMP_THREAD_LIMIT nproc)}" \
|
||||
--mount "type=bind,src=$output,dst=/results" \
|
||||
"$image" "$command" "$@" || status=$?
|
||||
profile=full
|
||||
if [[ "$command" = smoke || " $* " = *' --smoke '* ]]; then profile=smoke; fi
|
||||
if [[ -f "$output/$profile/index.html" ]]; then
|
||||
printf '\nOpen %s/%s/index.html\n' "$output" "$profile"
|
||||
fi
|
||||
exit "$status"
|
||||
;;
|
||||
*)
|
||||
echo 'Usage: bash artifact.sh {build|smoke|reproduce|plots|package}'
|
||||
[[ "$command" = help ]]
|
||||
;;
|
||||
esac
|
||||
@@ -0,0 +1,75 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
source /opt/artifact/pins.env
|
||||
export CMAKE_BUILD_PARALLEL_LEVEL=${1:-$(env -u OMP_NUM_THREADS -u OMP_THREAD_LIMIT nproc)}
|
||||
[[ "$CMAKE_BUILD_PARALLEL_LEVEL" =~ ^[1-9][0-9]*$ ]]
|
||||
cd /opt/raptor
|
||||
sha256sum -c /opt/artifact/models.sha256
|
||||
|
||||
checkout() {
|
||||
local url=$1 revision=$2 destination=$3
|
||||
git init "$destination"
|
||||
git -C "$destination" remote add origin "$url"
|
||||
git -C "$destination" fetch --depth 1 origin "$revision"
|
||||
git -C "$destination" checkout --detach FETCH_HEAD
|
||||
test "$(git -C "$destination" rev-parse HEAD)" = "$revision"
|
||||
git -C "$destination" submodule update --init --recursive --depth 1
|
||||
}
|
||||
|
||||
checkout https://github.com/onnx/onnx-mlir.git "$ONNX_MLIR_COMMIT" onnx-mlir
|
||||
checkout https://github.com/llvm/llvm-project.git "$LLVM_COMMIT" onnx-mlir/llvm-project
|
||||
checkout https://github.com/HEAPLab/PIMCOMP-NN.git "$PIMCOMP_COMMIT" third_party/PIMCOMP-NN
|
||||
checkout https://github.com/HEAPLab/pimsim-nn.git "$PIMSIM_COMMIT" backend-simulators/pim/pimsim-nn
|
||||
checkout https://github.com/protocolbuffers/protobuf.git "$PROTOBUF_COMMIT" /opt/protobuf
|
||||
|
||||
common=(-G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_POSITION_INDEPENDENT_CODE=ON
|
||||
-DCMAKE_EXE_LINKER_FLAGS=-fuse-ld=mold
|
||||
-DCMAKE_SHARED_LINKER_FLAGS=-fuse-ld=mold
|
||||
-DCMAKE_MODULE_LINKER_FLAGS=-fuse-ld=mold)
|
||||
|
||||
cmake -S /opt/protobuf -B /opt/protobuf/build "${common[@]}" \
|
||||
-Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_ABSL_PROVIDER=module \
|
||||
-DABSL_PROPAGATE_CXX_STD=ON -DCMAKE_CXX_STANDARD=17
|
||||
cmake --build /opt/protobuf/build
|
||||
cmake --install /opt/protobuf/build
|
||||
ldconfig
|
||||
|
||||
llvm=onnx-mlir/llvm-project
|
||||
cmake -S "$llvm/llvm" -B "$llvm/build_release" "${common[@]}" \
|
||||
'-DLLVM_ENABLE_PROJECTS=mlir;clang' -DLLVM_ENABLE_RUNTIMES=openmp \
|
||||
-DLLVM_TARGETS_TO_BUILD=host -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_ENABLE_RTTI=ON \
|
||||
-DENABLE_LIBOMPTARGET=OFF -DLLVM_ENABLE_LIBEDIT=OFF \
|
||||
-DLLVM_PARALLEL_LINK_JOBS="$CMAKE_BUILD_PARALLEL_LEVEL"
|
||||
cmake --build "$llvm/build_release"
|
||||
|
||||
cmake -S . -B build_release "${common[@]}" \
|
||||
-DONNX_MLIR_ACCELERATORS=PIM -DLLVM_ENABLE_ASSERTIONS=ON \
|
||||
-DONNX_MLIR_ENABLE_JAVA=OFF -DONNX_MLIR_BUILD_TESTS=OFF \
|
||||
-DMLIR_DIR="$PWD/$llvm/build_release/lib/cmake/mlir"
|
||||
cmake --build ./build_release
|
||||
|
||||
cmake -S third_party/PIMCOMP-NN -B third_party/PIMCOMP-NN/build "${common[@]}"
|
||||
cmake --build third_party/PIMCOMP-NN/build --target PIMCOMP-NN
|
||||
sim=backend-simulators/pim/pimsim-nn
|
||||
cmake -S "$sim" -B "$sim/build" "${common[@]}"
|
||||
cmake --build "$sim/build"
|
||||
ctest --test-dir "$sim/build" --output-on-failure
|
||||
|
||||
crate=backend-simulators/pim/pim-simulator
|
||||
cargo build --locked --release --no-default-features --manifest-path "$crate/Cargo.toml" \
|
||||
--package pim-simulator --bin pim-simulator --jobs "$CMAKE_BUILD_PARALLEL_LEVEL"
|
||||
# The experiment driver uses cargo run. All dependencies must be available offline.
|
||||
cargo fetch --locked --manifest-path "$crate/Cargo.toml"
|
||||
mkdir -p /opt/artifact/provenance
|
||||
cp /opt/artifact/pins.env /opt/artifact/source.sha256 /opt/artifact/models.sha256 /opt/artifact/provenance/
|
||||
for dir in onnx-mlir "$llvm" third_party/PIMCOMP-NN "$sim" /opt/protobuf; do
|
||||
printf '%s %s\n' "$dir" "$(git -C "$dir" rev-parse HEAD)"
|
||||
git -C "$dir" submodule status --recursive
|
||||
done > /opt/artifact/provenance/revisions.txt
|
||||
dpkg-query -W > /opt/artifact/provenance/system-packages.txt
|
||||
.venv/bin/python -m pip freeze > /opt/artifact/provenance/python-packages.txt
|
||||
{ clang-19 --version; mold --version; cmake --version; rustc --version; } \
|
||||
> /opt/artifact/provenance/toolchain.txt
|
||||
# Runtime uses an arbitrary host UID. Cargo and the runners need writable caches
|
||||
# and ONNX-MLIR writes temporary shims/configurations inside its source tree.
|
||||
chmod -R a+rwX /opt/raptor /opt/cargo /opt/rustup
|
||||
@@ -0,0 +1,5 @@
|
||||
788088b908e233d924c7c26b997e89ee861290c7bc56783a306e8201d79aac8f validation/networks/pimcomp_models/resnet18/resnet18-v1-7.onnx
|
||||
c3231061d081bdd47884137b02134f85142752a39e87263c529cd14ed242b096 validation/networks/pimcomp_models/resnet34/resnet34-v1-7.onnx
|
||||
c960c195953423388733cd0e60712cf425ca1f8917e264f5cf98e028752c61cf validation/networks/pimcomp_models/googlenet/googlenet-12-no-lrn.onnx
|
||||
396cdea21e5e7d02c3f26f14d22ef20975171702493f5c5e79b8e0d896e541ef validation/networks/pimcomp_models/vgg8/vgg8-mnist-reconstructed.onnx
|
||||
5e7bd85dc60b252674f8d7e1ad24f2317d00dfb8c524826536698462de04620d validation/networks/pimcomp_models/yolo11n/yolo11n.onnx
|
||||
@@ -0,0 +1,8 @@
|
||||
# Raptor main at artifact preparation time. Do not replace this with a branch.
|
||||
RAPTOR_COMMIT=4a1fe710b2f0e4e9c0504f4fd9b4c4dd3df53b7d
|
||||
ONNX_MLIR_COMMIT=eb54c2afc46d00c6b196d1f275b6bfee17e12f69
|
||||
LLVM_COMMIT=0c2701fe7fa002e1befc5f86c268a7964f96d286
|
||||
PIMCOMP_COMMIT=7b6be6da4d33bf19f68d073eccac2a78e326a5af
|
||||
PIMSIM_COMMIT=17e37c15da5473362e53268dabdd07dff5fe8fbf
|
||||
PROTOBUF_COMMIT=6a6cd88c262ffdb1738167a47d5fcc7a3eb4edac
|
||||
RUST_VERSION=1.90.0
|
||||
@@ -0,0 +1,59 @@
|
||||
"""Plot planning and synchronization gains without hiding invalid programs."""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import math
|
||||
from pathlib import Path
|
||||
|
||||
import matplotlib
|
||||
matplotlib.use('Agg')
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('--input', type=Path, required=True)
|
||||
parser.add_argument('--output-dir', type=Path, required=True)
|
||||
args = parser.parse_args()
|
||||
with args.input.open(newline='') as stream:
|
||||
rows = list(csv.DictReader(stream))
|
||||
if not rows:
|
||||
raise ValueError('The ablation CSV is empty')
|
||||
panels = [('plan', 'latency', 'latency_gain_percent', 'Planning: latency reduction (%)'),
|
||||
('plan', 'throughput', 'throughput_gain_percent', 'Planning: throughput increase (%)'),
|
||||
('sync', 'throughput', 'throughput_gain_percent', 'Synchronization: throughput increase (%)')]
|
||||
fig, axes = plt.subplots(3, 1, figsize=(11, 9), layout='constrained')
|
||||
for ax, (comparison, mode, field, label) in zip(axes, panels):
|
||||
selected = sorted((r for r in rows if r['comparison'] == comparison and r['mode'] == mode),
|
||||
key=lambda r: (r['arch'], r['model'], int(r['raptor_pipeline'])))
|
||||
for x, row in enumerate(selected):
|
||||
try:
|
||||
value = float(row[field].rstrip('%'))
|
||||
except (ValueError, TypeError):
|
||||
value = math.nan
|
||||
valid = row['functional_validation'] == row['baseline_functional_validation'] == 'PASS'
|
||||
if math.isfinite(value):
|
||||
ax.bar(x, value, color='#B3D5CA' if valid else '#FF9494',
|
||||
edgecolor='#222222', hatch='' if valid else '//')
|
||||
ax.annotate(f'{value:.1f}' + ('' if valid else ' INVALID'),
|
||||
(x, value), xytext=(0, 4 if value >= 0 else -4),
|
||||
textcoords='offset points', ha='center',
|
||||
va='bottom' if value >= 0 else 'top', fontsize=7)
|
||||
else:
|
||||
ax.text(x, 0, 'NA', ha='center', va='bottom', fontsize=8)
|
||||
ax.set_xticks(range(len(selected)),
|
||||
[f"{r['model']}\n{r['arch']} p{r['raptor_pipeline']}" for r in selected], fontsize=8)
|
||||
ax.set_ylabel(label)
|
||||
ax.axhline(0, color='#222222', linewidth=.7)
|
||||
ax.set_xlim(-.6, max(.6, len(selected) - .4))
|
||||
ax.margins(y=.3)
|
||||
fig.suptitle('Raptor ablation (red hatching: functional validation failed)')
|
||||
args.output_dir.mkdir(parents=True, exist_ok=True)
|
||||
for extension in ('pdf', 'svg', 'png'):
|
||||
fig.savefig(args.output_dir / f'ablation.{extension}', dpi=200)
|
||||
plt.close(fig)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
@@ -0,0 +1,39 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
cd -- "$(dirname -- "${BASH_SOURCE[0]}")"
|
||||
source pins.env
|
||||
repo=$(git -C .. rev-parse --show-toplevel)
|
||||
test "$(git -C "$repo" rev-parse "$RAPTOR_COMMIT^{commit}")" = "$RAPTOR_COMMIT"
|
||||
for item in "onnx-mlir:$ONNX_MLIR_COMMIT" \
|
||||
"third_party/PIMCOMP-NN:$PIMCOMP_COMMIT" \
|
||||
"backend-simulators/pim/pimsim-nn:$PIMSIM_COMMIT"; do
|
||||
path=${item%:*}
|
||||
expected=${item##*:}
|
||||
actual=$(git -C "$repo" rev-parse "$RAPTOR_COMMIT:$path")
|
||||
test "$actual" = "$expected"
|
||||
done
|
||||
if [[ "${1:-}" != --overlay-only ]]; then
|
||||
# Include the complete compiler and evaluation code, plus only the suite models.
|
||||
# The last two paths are the targets of the suite's relative model symlinks.
|
||||
git -C "$repo" archive --format=tar "$RAPTOR_COMMIT" \
|
||||
CMakeLists.txt README.md .gitmodules src test docs backend-simulators \
|
||||
validation/README.md validation/validate.py validation/raptor_validation \
|
||||
validation/tools validation/pimsim_configs validation/networks/pimcomp_models \
|
||||
validation/networks/resnet18/depth_68/resnet18_depth_68.onnx \
|
||||
validation/networks/yolo11n/depth_51/yolo11n_depth_51.onnx \
|
||||
| gzip -n > source.tar.gz.tmp
|
||||
mv source.tar.gz.tmp source.tar.gz
|
||||
sha256sum source.tar.gz > source.sha256
|
||||
fi
|
||||
overlay=(validation/raptor_validation/experiment_scheduler.py
|
||||
validation/tools/pim/pimcomp/compare/run_pimcomp_models.py
|
||||
validation/tools/pim/pimcomp/compare/compare_raptor_pimcomp_model.py
|
||||
validation/tools/pim/ablation/run_ablation.py
|
||||
validation/tools/pim/pimcomp/compare/README.md
|
||||
validation/tools/pim/ablation/README.md)
|
||||
for path in "${overlay[@]}"; do
|
||||
mkdir -p "validation-overlay/$(dirname "$path")"
|
||||
cp "$repo/$path" "validation-overlay/$path"
|
||||
done
|
||||
sha256sum "${overlay[@]/#/validation-overlay/}" > validation-overlay.sha256
|
||||
printf 'Prepared Raptor %s\n' "$RAPTOR_COMMIT"
|
||||
@@ -0,0 +1,23 @@
|
||||
# Evaluation and ONNX-MLIR build dependencies. The Graph Explorer is not required.
|
||||
cmake==3.31.6
|
||||
colorama==0.4.6
|
||||
contourpy==1.3.2
|
||||
cycler==0.12.1
|
||||
execnet==2.1.1
|
||||
fonttools==4.58.0
|
||||
iniconfig==2.1.0
|
||||
kiwisolver==1.4.8
|
||||
lit==15.0.7
|
||||
matplotlib==3.10.3
|
||||
numpy==2.2.6
|
||||
onnx==1.17.0
|
||||
packaging==25.0
|
||||
pillow==11.2.1
|
||||
pluggy==1.5.0
|
||||
protobuf==7.34.0
|
||||
pyparsing==3.2.3
|
||||
pytest==8.3.5
|
||||
pytest-xdist==3.6.1
|
||||
python-dateutil==2.9.0.post0
|
||||
PyYAML==6.0.2
|
||||
six==1.17.0
|
||||
+140
@@ -0,0 +1,140 @@
|
||||
"""Container commands for the pinned Raptor paper artifact."""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
from datetime import datetime, timezone
|
||||
from html import escape
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import platform
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
REPO = Path('/opt/raptor')
|
||||
PYTHON = REPO / '.venv/bin/python'
|
||||
COMPARE = Path('validation/tools/pim/pimcomp/compare')
|
||||
ABLATION = Path('validation/tools/pim/ablation/run_ablation.py')
|
||||
|
||||
|
||||
def stage(name: str, command: list[str], root: Path) -> dict:
|
||||
print(f'[{name}] {command}', flush=True)
|
||||
start = time.monotonic()
|
||||
with (root / f'{name}.log').open('wb') as log:
|
||||
with subprocess.Popen(command, cwd=REPO, stdout=subprocess.PIPE,
|
||||
stderr=subprocess.STDOUT) as process:
|
||||
for line in process.stdout:
|
||||
sys.stdout.buffer.write(line)
|
||||
sys.stdout.buffer.flush()
|
||||
log.write(line)
|
||||
returncode = process.wait()
|
||||
return {'command': command, 'exit_code': returncode,
|
||||
'seconds': round(time.monotonic() - start, 3)}
|
||||
|
||||
|
||||
def plots(root: Path, smoke: bool) -> dict:
|
||||
commands = {'ablation-plots': [str(PYTHON), str(HERE / 'plot_ablation.py'),
|
||||
'--input', str(root / 'ablation/results_ablation.csv'),
|
||||
'--output-dir', str(root / 'plots/ablation')]}
|
||||
if not smoke:
|
||||
commands['comparison-plots'] = [str(PYTHON), str(COMPARE / 'generate_plots.py'),
|
||||
'--input', str(root / 'comparison/results_comparison.csv'),
|
||||
'--output-dir', str(root / 'plots/comparison')]
|
||||
return {name: stage(name, command, root) for name, command in commands.items()}
|
||||
|
||||
|
||||
def show_results(root: Path) -> None:
|
||||
page = ['<!doctype html><html lang="en"><meta charset="utf-8">',
|
||||
'<meta name="viewport" content="width=device-width, initial-scale=1">',
|
||||
'<title>Raptor results</title><style>',
|
||||
'body{font:16px sans-serif;max-width:1100px;margin:2rem auto;padding:0 1rem}',
|
||||
'img{max-width:100%}table{border-collapse:collapse;font-size:13px}',
|
||||
'th,td{padding:6px;border:1px solid #ccc;white-space:nowrap}',
|
||||
'.table{overflow-x:auto}</style><h1>Raptor results</h1>',
|
||||
'<p>Functional validation status is shown in the tables and plots.</p>']
|
||||
for title, relative in (
|
||||
('Comparison', 'plots/comparison/png/performance_energy_compact.png'),
|
||||
('Pipeline throughput', 'plots/comparison/png/raptor_pipeline_throughput.png'),
|
||||
('Ablation', 'plots/ablation/ablation.png'),
|
||||
):
|
||||
if (root / relative).is_file():
|
||||
page.append(f'<h2>{title}</h2><a href="{relative}"><img src="{relative}" alt="{title}"></a>')
|
||||
for title, relative in (('Comparison', 'comparison/results_comparison.csv'),
|
||||
('Ablation', 'ablation/results_ablation.csv')):
|
||||
path = root / relative
|
||||
if path.is_file():
|
||||
page.append(f'<h2>{title} data</h2><p><a href="{relative}">Download CSV</a></p><div class="table"><table>')
|
||||
with path.open(newline='') as stream:
|
||||
for index, row in enumerate(csv.reader(stream)):
|
||||
tag = 'th' if index == 0 else 'td'
|
||||
page.append('<tr>' + ''.join(f'<{tag}>{escape(value)}</{tag}>' for value in row) + '</tr>')
|
||||
page.append('</table></div>')
|
||||
page.append('<h2>Files</h2><ul>')
|
||||
for path in sorted([*root.glob('*.log'), *root.glob('plots/*')]):
|
||||
relative = path.relative_to(root).as_posix()
|
||||
page.append(f'<li><a href="{relative}">{escape(relative)}</a></li>')
|
||||
page.append('</ul></html>')
|
||||
(root / 'index.html').write_text('\n'.join(page) + '\n')
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument('command', choices=('reproduce', 'smoke', 'plots'))
|
||||
parser.add_argument('--smoke', action='store_true', help='Plot the smoke results.')
|
||||
parser.add_argument('--jobs', type=int, default=int(os.environ.get('JOBS', len(os.sched_getaffinity(0)))))
|
||||
parser.add_argument('--threads-per-job', type=int,
|
||||
default=int(os.environ.get('THREADS_PER_JOB', len(os.sched_getaffinity(0)))))
|
||||
args = parser.parse_args()
|
||||
smoke = args.command == 'smoke' or args.smoke
|
||||
root = Path('/results') / ('smoke' if smoke else 'full')
|
||||
os.environ['CARGO_NET_OFFLINE'] = 'true'
|
||||
if args.command == 'plots':
|
||||
manifest_path = root / 'run.json'
|
||||
manifest = json.loads(manifest_path.read_text())
|
||||
records = plots(root, smoke)
|
||||
manifest['stages'].update(records)
|
||||
manifest_path.write_text(json.dumps(manifest, indent=2) + '\n')
|
||||
show_results(root)
|
||||
return int(any(record['exit_code'] for record in records.values()))
|
||||
if root.exists():
|
||||
parser.error(f'{root} exists. Select a fresh OUTPUT_DIR to prevent stale result reuse.')
|
||||
jobs = args.jobs
|
||||
if jobs < 1 or args.threads_per_job < 1:
|
||||
parser.error('jobs and threads-per-job must be positive')
|
||||
root.mkdir(parents=True)
|
||||
shutil.copytree(HERE / 'provenance', root / 'provenance')
|
||||
# Copy the artifact scripts so each result identifies its orchestration code.
|
||||
shutil.copytree(HERE, root / 'provenance/artifact',
|
||||
ignore=shutil.ignore_patterns('provenance', '__pycache__'))
|
||||
manifest = {'started_utc': datetime.now(timezone.utc).isoformat(),
|
||||
'profile': 'smoke' if smoke else 'full', 'image_id': os.environ.get('ARTIFACT_IMAGE_ID'),
|
||||
'platform': platform.platform(), 'machine': platform.machine(),
|
||||
'cpu_count': os.cpu_count(), 'jobs': jobs, 'threads_per_job': args.threads_per_job,
|
||||
'batch_size': 64, 'fast': False, 'stages': {}}
|
||||
common = ['--jobs', str(jobs), '--threads-per-job', str(args.threads_per_job),
|
||||
'--batch-size', '64', '--no-fast']
|
||||
subset = ['--models', 'vgg8', '--archs', 'arch-a', '--mode', 'latency', '--pipeline', '1'] if smoke else []
|
||||
commands = {
|
||||
'experiments': [str(PYTHON), str(COMPARE / 'run_pimcomp_models.py'),
|
||||
'--out-dir', str(root / 'comparison'),
|
||||
'--with-ablation', '--ablation-out-dir', str(root / 'ablation'),
|
||||
'--common-dir', str(root / 'references'), *common, *subset],
|
||||
}
|
||||
manifest_path = root / 'run.json'
|
||||
manifest_path.write_text(json.dumps(manifest, indent=2) + '\n')
|
||||
for name, command in commands.items():
|
||||
manifest['stages'][name] = stage(name, command, root)
|
||||
manifest_path.write_text(json.dumps(manifest, indent=2) + '\n')
|
||||
manifest['stages'].update(plots(root, smoke))
|
||||
manifest['finished_utc'] = datetime.now(timezone.utc).isoformat()
|
||||
manifest_path.write_text(json.dumps(manifest, indent=2) + '\n')
|
||||
show_results(root)
|
||||
return int(any(record['exit_code'] for record in manifest['stages'].values()))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
raise SystemExit(main())
|
||||
@@ -0,0 +1,153 @@
|
||||
"""Run experiment subprocesses as their dependencies complete."""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
from collections import deque
|
||||
from dataclasses import dataclass
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
import signal
|
||||
import subprocess
|
||||
import time
|
||||
from typing import Callable
|
||||
|
||||
|
||||
def available_processors() -> int:
|
||||
if hasattr(os, "sched_getaffinity"):
|
||||
return len(os.sched_getaffinity(0))
|
||||
return os.cpu_count() or 1
|
||||
|
||||
|
||||
def positive_integer(value: str) -> int:
|
||||
number = int(value)
|
||||
if number < 1:
|
||||
raise argparse.ArgumentTypeError("must be positive")
|
||||
return number
|
||||
|
||||
|
||||
def thread_environment(threads: int) -> dict[str, str]:
|
||||
return {**os.environ, "OMP_NUM_THREADS": str(threads),
|
||||
"OMP_THREAD_LIMIT": str(threads), "OMP_DYNAMIC": "FALSE",
|
||||
"OMP_WAIT_POLICY": "PASSIVE", "OPENBLAS_NUM_THREADS": str(threads),
|
||||
"MKL_NUM_THREADS": str(threads), "RAYON_NUM_THREADS": str(threads),
|
||||
"CMAKE_BUILD_PARALLEL_LEVEL": str(threads), "CARGO_BUILD_JOBS": str(threads)}
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ExperimentTask:
|
||||
key: str
|
||||
command: Callable[[], list[str]]
|
||||
dependencies: tuple[str, ...]
|
||||
log_path: Path
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class TaskResult:
|
||||
returncode: int
|
||||
error: str = ""
|
||||
|
||||
|
||||
def stop_processes(processes: list[subprocess.Popen]) -> None:
|
||||
for process in processes:
|
||||
try:
|
||||
os.killpg(process.pid, signal.SIGTERM)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
deadline = time.monotonic() + 5
|
||||
for process in processes:
|
||||
try:
|
||||
process.wait(timeout=max(0, deadline - time.monotonic()))
|
||||
except subprocess.TimeoutExpired:
|
||||
try:
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
process.wait()
|
||||
|
||||
|
||||
def run_tasks(tasks: list[ExperimentTask], jobs: int, threads: int,
|
||||
cwd: Path) -> dict[str, TaskResult]:
|
||||
if jobs < 1 or threads < 1:
|
||||
raise ValueError("jobs and threads-per-job must be positive")
|
||||
by_key = {task.key: task for task in tasks}
|
||||
if len(by_key) != len(tasks):
|
||||
raise ValueError("duplicate experiment task")
|
||||
waiting = {task.key: set(task.dependencies) for task in tasks}
|
||||
children = {key: [] for key in by_key}
|
||||
for task in tasks:
|
||||
for dependency in task.dependencies:
|
||||
if dependency not in by_key:
|
||||
raise ValueError(f"unknown dependency: {dependency}")
|
||||
children[dependency].append(task.key)
|
||||
ready = deque(key for key in by_key if not waiting[key])
|
||||
degrees = {key: len(dependencies) for key, dependencies in waiting.items()}
|
||||
pending = deque(ready)
|
||||
visited = 0
|
||||
while pending:
|
||||
visited += 1
|
||||
for child in children[pending.popleft()]:
|
||||
degrees[child] -= 1
|
||||
if not degrees[child]:
|
||||
pending.append(child)
|
||||
if visited != len(tasks):
|
||||
raise ValueError("experiment dependency cycle")
|
||||
active = {}
|
||||
results = {}
|
||||
records = {}
|
||||
environment = thread_environment(threads)
|
||||
|
||||
def complete(key: str, result: TaskResult) -> None:
|
||||
results[key] = result
|
||||
task = by_key[key]
|
||||
record = records.get(key, {"dependencies": task.dependencies, "threads_per_job": threads})
|
||||
record.update(returncode=result.returncode, error=result.error)
|
||||
task.log_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
task.log_path.with_suffix(".execution.json").write_text(json.dumps(record, indent=2) + "\n")
|
||||
print(f"[{'Failed' if result.returncode else 'Completed'}] {key}"
|
||||
f"{': ' + result.error if result.error else ''}", flush=True)
|
||||
for child in children[key]:
|
||||
waiting[child].remove(key)
|
||||
if not waiting[child]:
|
||||
ready.append(child)
|
||||
|
||||
try:
|
||||
while ready or active:
|
||||
while ready and len(active) < jobs:
|
||||
key = ready.popleft()
|
||||
task = by_key[key]
|
||||
failed = [dep for dep in task.dependencies if results[dep].returncode]
|
||||
if failed:
|
||||
complete(key, TaskResult(1, "prerequisite failed: " + ", ".join(failed)))
|
||||
continue
|
||||
log = None
|
||||
try:
|
||||
command = task.command()
|
||||
task.log_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
log = task.log_path.open("wb")
|
||||
metadata = {"command": command, "threads_per_job": threads,
|
||||
"dependencies": task.dependencies}
|
||||
records[key] = metadata
|
||||
task.log_path.with_suffix(".execution.json").write_text(json.dumps(metadata, indent=2) + "\n")
|
||||
process = subprocess.Popen(command, cwd=cwd, env=environment,
|
||||
stdout=log, stderr=subprocess.STDOUT, start_new_session=True)
|
||||
except (OSError, ValueError, RuntimeError) as exc:
|
||||
if log is not None:
|
||||
log.close()
|
||||
complete(key, TaskResult(1, str(exc)))
|
||||
continue
|
||||
active[key] = process, log
|
||||
print(f"[Started] {key}; log: {task.log_path}", flush=True)
|
||||
completed = [(key, process.returncode) for key, (process, _) in active.items()
|
||||
if process.poll() is not None]
|
||||
for key, returncode in completed:
|
||||
_, log = active.pop(key)
|
||||
log.close()
|
||||
complete(key, TaskResult(returncode))
|
||||
if active and not completed:
|
||||
time.sleep(.05)
|
||||
return results
|
||||
finally:
|
||||
stop_processes([process for process, _ in active.values()])
|
||||
for _, log in active.values():
|
||||
log.close()
|
||||
@@ -0,0 +1,187 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
import pytest
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tools/pim/pimcomp/compare"))
|
||||
from raptor_validation import experiment_scheduler as scheduler
|
||||
import run_pimcomp_models as suite
|
||||
import compare_raptor_pimcomp_model as comparator
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tools/pim/ablation"))
|
||||
import run_ablation as ablation
|
||||
|
||||
|
||||
def task(root, key, code, dependencies=()):
|
||||
command = [sys.executable, "-c", code]
|
||||
return scheduler.ExperimentTask(key, lambda: command, dependencies, root / f"{key}.log")
|
||||
|
||||
|
||||
def test_dependency_starts_before_unrelated_job_finishes(tmp_path):
|
||||
# The slow job cannot finish until the fast job's dependent has run.
|
||||
marker = tmp_path / "released"
|
||||
slow = ("import pathlib,time; deadline=time.monotonic()+5; "
|
||||
f"p=pathlib.Path({str(marker)!r}); "
|
||||
"exec('while not p.exists() and time.monotonic()<deadline: time.sleep(.01)'); "
|
||||
"assert p.exists(), 'dependent was blocked by an unrelated job'")
|
||||
tasks = [task(tmp_path, "fast", "pass"), task(tmp_path, "slow", slow),
|
||||
task(tmp_path, "dependent", f"from pathlib import Path; Path({str(marker)!r}).touch()", ("fast",))]
|
||||
results = scheduler.run_tasks(tasks, 2, 1, tmp_path)
|
||||
assert all(result.returncode == 0 for result in results.values())
|
||||
|
||||
|
||||
def test_failure_skips_dependents_but_runs_other_jobs(tmp_path):
|
||||
marker = tmp_path / "must-not-run"
|
||||
tasks = [task(tmp_path, "failed", "raise SystemExit(3)"),
|
||||
task(tmp_path, "dependent", f"from pathlib import Path; Path({str(marker)!r}).touch()", ("failed",)),
|
||||
task(tmp_path, "independent", "pass")]
|
||||
results = scheduler.run_tasks(tasks, 2, 1, tmp_path)
|
||||
assert results["failed"].returncode == 3
|
||||
assert results["dependent"].returncode != 0
|
||||
assert results["independent"].returncode == 0
|
||||
assert not marker.exists()
|
||||
|
||||
|
||||
def test_thread_environment_and_execution_record(tmp_path):
|
||||
names = ["OMP_NUM_THREADS", "OMP_THREAD_LIMIT", "RAYON_NUM_THREADS",
|
||||
"CMAKE_BUILD_PARALLEL_LEVEL", "CARGO_BUILD_JOBS"]
|
||||
job = task(tmp_path, "environment", f"import os,json; print(json.dumps({{k:os.environ[k] for k in {names!r}}}))")
|
||||
scheduler.run_tasks([job], 1, 3, tmp_path)
|
||||
assert set(json.loads(job.log_path.read_text()).values()) == {"3"}
|
||||
record = json.loads(job.log_path.with_suffix(".execution.json").read_text())
|
||||
assert record["threads_per_job"] == 3
|
||||
assert scheduler.thread_environment(3)["OMP_WAIT_POLICY"] == "PASSIVE"
|
||||
|
||||
|
||||
def test_concurrent_process_limit(tmp_path):
|
||||
jobs = []
|
||||
for index in range(6):
|
||||
output = tmp_path / f"interval-{index}.json"
|
||||
code = ("import json,time; from pathlib import Path; start=time.monotonic_ns(); "
|
||||
f"time.sleep(.05); Path({str(output)!r}).write_text(json.dumps([start,time.monotonic_ns()]))")
|
||||
jobs.append(task(tmp_path, str(index), code))
|
||||
results = scheduler.run_tasks(jobs, 2, 1, tmp_path)
|
||||
assert all(result.returncode == 0 for result in results.values())
|
||||
events = []
|
||||
for path in tmp_path.glob("interval-*.json"):
|
||||
start, finish = json.loads(path.read_text())
|
||||
events.extend(((start, 1), (finish, -1)))
|
||||
active = 0
|
||||
for _, delta in sorted(events):
|
||||
active += delta
|
||||
assert active <= 2
|
||||
assert len(events) == 12 and active == 0
|
||||
|
||||
|
||||
def test_interrupt_terminates_workers(tmp_path, monkeypatch):
|
||||
processes = []
|
||||
popen = scheduler.subprocess.Popen
|
||||
|
||||
def start(*args, **kwargs):
|
||||
process = popen(*args, **kwargs)
|
||||
processes.append(process)
|
||||
return process
|
||||
|
||||
monkeypatch.setattr(scheduler.subprocess, "Popen", start)
|
||||
sleep = scheduler.time.sleep
|
||||
interrupted = False
|
||||
|
||||
def interrupt_once(seconds):
|
||||
nonlocal interrupted
|
||||
if not interrupted:
|
||||
interrupted = True
|
||||
raise KeyboardInterrupt()
|
||||
sleep(seconds)
|
||||
|
||||
monkeypatch.setattr(scheduler.time, "sleep", interrupt_once)
|
||||
with pytest.raises(KeyboardInterrupt):
|
||||
scheduler.run_tasks([task(tmp_path, "waiting", "import time; time.sleep(30)")], 1, 1, tmp_path)
|
||||
assert processes and all(process.poll() is not None for process in processes)
|
||||
|
||||
|
||||
def test_invalid_dependency_graph_is_rejected(tmp_path):
|
||||
with pytest.raises(ValueError, match="unknown dependency"):
|
||||
scheduler.run_tasks([task(tmp_path, "a", "pass", ("missing",))], 1, 1, tmp_path)
|
||||
with pytest.raises(ValueError, match="cycle"):
|
||||
scheduler.run_tasks([task(tmp_path, "a", "pass", ("b",)),
|
||||
task(tmp_path, "b", "pass", ("a",))], 1, 1, tmp_path)
|
||||
|
||||
|
||||
def test_defaults_follow_processor_count(monkeypatch):
|
||||
monkeypatch.setattr(suite, "available_processors", lambda: 7)
|
||||
args = suite.parse_args([])
|
||||
assert args.jobs == args.threads_per_job == 7
|
||||
args = suite.parse_args(["--jobs", "2", "--threads-per-job", "3"])
|
||||
assert (args.jobs, args.threads_per_job) == (2, 3)
|
||||
with pytest.raises(SystemExit):
|
||||
suite.parse_args(["--threads-per-job", "0"])
|
||||
monkeypatch.setattr(ablation, "available_processors", lambda: 7)
|
||||
args, forwarded = ablation.parse_args([])
|
||||
assert args.jobs == args.threads_per_job == 7
|
||||
assert forwarded[forwarded.index("--threads-per-job") + 1] == "7"
|
||||
|
||||
|
||||
def test_combined_plan_reuses_references_and_raptor_baselines(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(suite, "validate_pimcomp_source", lambda: None)
|
||||
args = suite.parse_args(["--out-dir", str(tmp_path / "comparison"),
|
||||
"--ablation-out-dir", str(tmp_path / "ablation"), "--common-dir", str(tmp_path / "references"),
|
||||
"--jobs", "2", "--threads-per-job", "2", "--dry-run"])
|
||||
comparison = suite.make_plan(args)
|
||||
ablations, _, _ = suite.combined_ablation_plans(args)
|
||||
tasks = suite.build_tasks([comparison, *ablations])
|
||||
assert len([job for job in tasks if job.key.startswith("reference:")]) == 5
|
||||
assert len(tasks) == 5 + 40 + 24 # 16 baseline cases reuse comparison reports.
|
||||
assert sum(len(plan.aliases) for plan in ablations) == 16
|
||||
assert len({job.key for job in tasks}) == len(tasks)
|
||||
for job in tasks:
|
||||
if job.key.startswith("reference:"):
|
||||
command = job.command()
|
||||
assert command[command.index("--jobs") + 1] == "2"
|
||||
continue
|
||||
command = job.command()
|
||||
assert command[command.index("--threads-per-job") + 1] == "2"
|
||||
if "--reuse-pimcomp-report" in command:
|
||||
assert len(job.dependencies) == 2
|
||||
else:
|
||||
assert len(job.dependencies) == 1
|
||||
|
||||
|
||||
def test_raptor_receives_thread_limit(tmp_path, monkeypatch):
|
||||
received = []
|
||||
monkeypatch.setattr(comparator, "compile_with_raptor",
|
||||
lambda *args, **kwargs: received.extend(kwargs["raptor_extra_args"]) or {})
|
||||
args = argparse.Namespace(threads_per_job=3, pimcomp_config=tmp_path / "config.json",
|
||||
raptor_extra_arg=[], raptor_path=tmp_path / "compiler", verbose_raptor_compile=False, timeout_seconds=0)
|
||||
comparator.compile_raptor_target(tmp_path / "model.onnx", tmp_path / "output",
|
||||
{"core_count": 1, "crossbar_count": 1, "crossbar_size": 128}, args, [])
|
||||
assert "-j=3" in received
|
||||
|
||||
|
||||
def test_stale_report_is_not_published_after_missing_execution(tmp_path):
|
||||
args = suite.parse_args(["--out-dir", str(tmp_path), "--models", "vgg8",
|
||||
"--archs", "arch-a", "--mode", "latency", "--raptor-only"])
|
||||
plan = suite.make_plan(args)
|
||||
report = suite.report_for(plan, plan.specs[0])
|
||||
report.parent.mkdir(parents=True)
|
||||
report.write_text(json.dumps({"validation_policy": suite.VALIDATION_POLICY,
|
||||
"raptor_validation": {"status": "done", "passed": True},
|
||||
"raptor_performance": {"latency_ms": 1}}))
|
||||
tasks = suite.build_tasks([plan])
|
||||
assert suite.finish_plan(plan, {task.key: suite.TaskResult(0) for task in tasks}) == 1
|
||||
import csv
|
||||
with (tmp_path / "results_comparison.csv").open() as stream:
|
||||
row = next(csv.DictReader(stream))
|
||||
assert row["raptor_latency_ms"] == row["raptor_functional_validation"] == "NA"
|
||||
|
||||
|
||||
def test_reference_workers_do_not_create_nested_thread_teams(tmp_path, monkeypatch):
|
||||
environments = []
|
||||
monkeypatch.setattr(comparator, "save_inputs_to_files", lambda *a: ([], None))
|
||||
monkeypatch.setattr(comparator, "run_logged", lambda *a, **k: environments.append(k["env"]))
|
||||
comparator.generate_reference_outputs(tmp_path / "runner", tmp_path, tmp_path / "model.onnx", [], [],
|
||||
argparse.Namespace(timeout_seconds=0), tmp_path / "output")
|
||||
assert environments[0]["OMP_NUM_THREADS"] == environments[0]["OPENBLAS_NUM_THREADS"] == "1"
|
||||
@@ -524,23 +524,32 @@ def test_suite_reruns_anchor_and_shares_only_new_reports(tmp_path, monkeypatch,
|
||||
report.parent.mkdir(parents=True)
|
||||
report.write_text("{}")
|
||||
os.utime(report, ns=(1, 1))
|
||||
batches = []
|
||||
commands = []
|
||||
errors = []
|
||||
|
||||
def run_jobs(jobs, max_workers, log_dir, offset):
|
||||
assert max_workers == 1
|
||||
batches.append([command for _, command, _ in jobs])
|
||||
for _, command, _ in jobs:
|
||||
output = Path(command[command.index("--out-dir") + 1]) / "pimcomp/comparison_report.json"
|
||||
if fresh or len(batches) > 1:
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
output.write_text('{"new": true}')
|
||||
return [], offset + len(jobs)
|
||||
def run_jobs(tasks, jobs, threads, cwd):
|
||||
assert jobs == 1
|
||||
results = {}
|
||||
for task in tasks:
|
||||
try:
|
||||
command = task.command()
|
||||
except ValueError as exc:
|
||||
errors.append(str(exc))
|
||||
results[task.key] = suite.TaskResult(1, str(exc))
|
||||
continue
|
||||
if "--prepare-common" not in command:
|
||||
commands.append(command)
|
||||
output = Path(command[command.index("--out-dir") + 1]) / "pimcomp/comparison_report.json"
|
||||
if fresh or only == "raptor":
|
||||
output.parent.mkdir(parents=True, exist_ok=True)
|
||||
output.write_text('{"new": true}')
|
||||
results[task.key] = suite.TaskResult(0)
|
||||
return results
|
||||
|
||||
summary = tmp_path / "results.csv"
|
||||
summary.write_text("summary\n")
|
||||
monkeypatch.setattr(suite, "validate_pimcomp_source", lambda: None)
|
||||
monkeypatch.setattr(suite, "run", lambda *a, **k: 0)
|
||||
monkeypatch.setattr(suite, "run_comparison_jobs", run_jobs)
|
||||
monkeypatch.setattr(suite, "run_tasks", run_jobs)
|
||||
monkeypatch.setattr(suite, "write_results_csv", lambda *a: summary)
|
||||
monkeypatch.setattr(suite, "comparison_passed", lambda report: True)
|
||||
monkeypatch.setattr(sys, "argv", [
|
||||
@@ -548,15 +557,19 @@ def test_suite_reruns_anchor_and_shares_only_new_reports(tmp_path, monkeypatch,
|
||||
"--archs", "arch-a", "--mode", "throughput", "--jobs", "1",
|
||||
*(["--only", only] if only else []),
|
||||
])
|
||||
assert suite.main() == 0
|
||||
assert suite.main() == (1 if only != "raptor" and not fresh else 0)
|
||||
if only == "raptor":
|
||||
assert len(batches[0]) == 3
|
||||
assert all("--reuse-pimcomp-report" in command for command in batches[0])
|
||||
assert len(commands) == 3
|
||||
assert all("--reuse-pimcomp-report" in command for command in commands)
|
||||
else:
|
||||
assert len(batches[0]) == 1
|
||||
assert "--reuse-pimcomp-report" not in batches[0][0]
|
||||
assert all(("--reuse-pimcomp-report" in command) == fresh for command in batches[1])
|
||||
assert ("--reuse-raptor-report" in batches[0][0]) == (only == "pimcomp")
|
||||
assert "--reuse-pimcomp-report" not in commands[0]
|
||||
assert ("--reuse-raptor-report" in commands[0]) == (only == "pimcomp")
|
||||
if fresh:
|
||||
assert len(commands) == 3
|
||||
assert all("--reuse-pimcomp-report" in command for command in commands[1:])
|
||||
else:
|
||||
assert len(commands) == 1 # Missing reports must not cause duplicate Pimcomp work.
|
||||
assert len(errors) == 2
|
||||
|
||||
|
||||
def test_missing_pimcomp_report_is_not_reused(tmp_path):
|
||||
@@ -658,7 +671,7 @@ def test_ablation_cli_selects_canonical_or_custom_output(tmp_path, monkeypatch,
|
||||
canonical = tmp_path / "scripts/results_ablation.csv"
|
||||
monkeypatch.setattr(ablation, "DEFAULT_OUT_DIR", artifact_root)
|
||||
monkeypatch.setattr(ablation, "DEFAULT_RESULTS_PATH", canonical)
|
||||
monkeypatch.setattr(ablation.subprocess, "Popen", lambda *a, **k: argparse.Namespace(wait=lambda: 0))
|
||||
monkeypatch.setattr(suite, "run_plans", lambda plans: 0)
|
||||
calls = []
|
||||
monkeypatch.setattr(ablation, "aggregate_results", lambda root, *a, **k: (calls.append((root, k["output"])) or None, []))
|
||||
command = ["--variant", "baseline", "--jobs", "1"]
|
||||
@@ -863,7 +876,13 @@ def test_ablation_runner_distinguishes_invalid_outputs_from_failed_execution(tmp
|
||||
csv = tmp_path / "results.csv"
|
||||
csv.write_text("summary\n")
|
||||
monkeypatch.setattr(suite, "run", lambda *a, **k: 0)
|
||||
monkeypatch.setattr(suite, "run_comparison_jobs", lambda *a: (["arch-a/vgg8/throughput/pipeline4"] if process_failed else [], 1))
|
||||
def complete_tasks(tasks, *args):
|
||||
previous = output.stat().st_mtime_ns
|
||||
output.write_text(json.dumps(report))
|
||||
os.utime(output, ns=(previous + 1000000, previous + 1000000))
|
||||
return {task.key: suite.TaskResult(int(process_failed)) for task in tasks}
|
||||
|
||||
monkeypatch.setattr(suite, "run_tasks", complete_tasks)
|
||||
monkeypatch.setattr(suite, "write_results_csv", lambda *a: csv)
|
||||
monkeypatch.setattr(sys, "argv", ["run_pimcomp_models.py", "--out-dir", str(tmp_path),
|
||||
"--models", "vgg8", "--archs", "arch-a", "--mode", "throughput", "--pipeline", "4",
|
||||
|
||||
@@ -9,6 +9,9 @@ Run the synchronization/Spatial-planning ablation from the repository root:
|
||||
Defaults: `vgg8 resnet18 resnet34 googlenet`, `arch-a arch-b`, latency
|
||||
(pipeline 1), and throughput (pipeline 4). Select `yolo11n` or `arch-c`
|
||||
explicitly when needed. Only Raptor runs; Pimcomp validation is `SKIPPED`.
|
||||
Ready models, architectures, modes, and variants run concurrently in one dependency queue.
|
||||
References are prepared once per model. `--jobs` and `--threads-per-job` both default
|
||||
to the available logical processor count; override them to limit CPU contention.
|
||||
|
||||
| Variant | Raptor options |
|
||||
|---|---|
|
||||
@@ -34,7 +37,7 @@ suite pipelines for that mode. Pipeline-only selection uses its matching mode.
|
||||
`--pipeline auto` requests automatic throughput depth selection; positive numeric
|
||||
depths retain their exact-stage meaning.
|
||||
`--only` is rejected because
|
||||
the wrapper always runs Raptor only. Ctrl+C stops the active variant's workers.
|
||||
the wrapper always runs Raptor only. Ctrl+C stops all active experiment workers.
|
||||
|
||||
Artifacts use `<out-dir>/<model>/artifacts/<arch>/<mode>[/pipelineN]/ablation/<variant>/`;
|
||||
reference inputs, outputs, and runners are shared under `artifacts/common/`.
|
||||
|
||||
@@ -6,10 +6,7 @@ from __future__ import annotations
|
||||
import argparse
|
||||
import csv
|
||||
import math
|
||||
import os
|
||||
import shlex
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
@@ -26,6 +23,7 @@ from raptor_validation.pimcomp_models import (
|
||||
)
|
||||
from raptor_validation.artifacts import remove_lock_files
|
||||
from raptor_validation.raptor import parse_pipeline_depth, pipeline_sort_key
|
||||
from raptor_validation.experiment_scheduler import available_processors, positive_integer
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -79,9 +77,14 @@ def parse_args(argv: list[str] | None = None) -> tuple[argparse.Namespace, list[
|
||||
help="Run only this variant; baseline is added when needed. Repeat as needed.",
|
||||
)
|
||||
parser.add_argument("--dry-run", action="store_true", help="Print runner commands without modifying files.")
|
||||
parser.add_argument("--jobs", "-j", type=positive_integer, default=available_processors(),
|
||||
help="Maximum concurrent cases (default: available logical processors).")
|
||||
parser.add_argument("--threads-per-job", type=positive_integer, default=available_processors(),
|
||||
help="Threads per case (default: available logical processors).")
|
||||
args, forwarded = parser.parse_known_args(argv)
|
||||
if any(option == "--only" or option.startswith("--only=") for option in forwarded):
|
||||
parser.error("--only is not supported; the ablation wrapper always runs Raptor only")
|
||||
forwarded.extend(("--jobs", str(args.jobs), "--threads-per-job", str(args.threads_per_job)))
|
||||
return args, forwarded
|
||||
|
||||
|
||||
@@ -156,23 +159,6 @@ def runner_command(
|
||||
return command
|
||||
|
||||
|
||||
def terminate_process_group(process: subprocess.Popen[bytes]) -> None:
|
||||
if process.poll() is not None:
|
||||
return
|
||||
try:
|
||||
os.killpg(process.pid, signal.SIGTERM)
|
||||
except ProcessLookupError:
|
||||
return
|
||||
try:
|
||||
process.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
try:
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
return
|
||||
process.wait()
|
||||
|
||||
|
||||
def percentage_difference(
|
||||
value: str | None, reference: str | None, *, lower_is_better: bool = False
|
||||
) -> str:
|
||||
@@ -313,33 +299,27 @@ def main(argv: list[str] | None = None) -> int:
|
||||
)
|
||||
return 0
|
||||
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
failed = []
|
||||
current_process = None
|
||||
sys.path.insert(0, str(RUNNER.parent))
|
||||
import run_pimcomp_models as suite
|
||||
plans = []
|
||||
try:
|
||||
for variant in variants:
|
||||
for case_label, case_forwarded in case_sets:
|
||||
for _, case_forwarded in case_sets:
|
||||
if variant.name == "no-sync" and case_key(case_forwarded)[0] != "throughput":
|
||||
continue
|
||||
command = runner_command(
|
||||
out_dir, variant, args.models, case_forwarded, out_dir, False
|
||||
)
|
||||
print(f"[{variant.name}/{case_label}] {shlex.join(command)}")
|
||||
current_process = subprocess.Popen(
|
||||
command,
|
||||
cwd=REPO,
|
||||
start_new_session=True,
|
||||
)
|
||||
returncode = current_process.wait()
|
||||
current_process = None
|
||||
if returncode:
|
||||
failed.append(f"{variant.name}/{case_label}: runner exited with {returncode}")
|
||||
command = runner_command(out_dir, variant, args.models, case_forwarded, out_dir, False)
|
||||
selected = suite.parse_args(command[2:])
|
||||
if selected.clean:
|
||||
print(f"Removed {suite.clean_artifacts(out_dir, args.models, out_dir)} artifact path(s).")
|
||||
return 0
|
||||
plans.append(suite.make_plan(selected))
|
||||
failed = ["experiment jobs failed"] if suite.run_plans(plans) else []
|
||||
except KeyboardInterrupt:
|
||||
if current_process is not None:
|
||||
terminate_process_group(current_process)
|
||||
remove_lock_files(out_dir)
|
||||
print("Interrupted; terminated the active ablation job.", file=sys.stderr)
|
||||
print("Interrupted; terminated ablation workers.", file=sys.stderr)
|
||||
return 130
|
||||
except ValueError as exc:
|
||||
print(str(exc), file=sys.stderr)
|
||||
return 2
|
||||
|
||||
remove_lock_files(out_dir)
|
||||
selected_cases = frozenset(case_key(arguments) for _, arguments in case_sets)
|
||||
|
||||
@@ -11,7 +11,12 @@ built release Raptor, Pimcomp, and `pimsim-nn`.
|
||||
```
|
||||
|
||||
Defaults: all five models, Arch-A/B, latency pipeline 1, throughput pipelines
|
||||
2/4/8, four workers, and a functional throughput batch of 64.
|
||||
2/4/8 and a functional throughput batch of 64.
|
||||
Both `--jobs` and `--threads-per-job` default to the available logical processor count.
|
||||
Their limits multiply: 40 jobs with 40 threads can create 1,600 threads.
|
||||
Use `--jobs 8 --threads-per-job 4` to reduce contention.
|
||||
The thread limit controls OpenMP, Raptor's compilation thread pool (`-j`), BLAS, and Rayon.
|
||||
Reference batches use that many workers, with one thread per reference process.
|
||||
Use `--help` for the complete option reference. Numeric depths greater than one
|
||||
require exactly that many active compute stages. `--pipeline auto` requests
|
||||
throughput depth selection. Reports include `raptor_selected_pipeline`; the
|
||||
@@ -26,6 +31,8 @@ requested depth remains in the existing pipeline field.
|
||||
| `--only {raptor,pimcomp}` | Rerun one compiler and reuse the other's existing report. |
|
||||
| `--raptor-only` | Run Raptor without invoking Pimcomp. |
|
||||
| `--batch-size N`, `--jobs N` | Set functional throughput batch size and worker count. |
|
||||
| `--threads-per-job N` | Set the thread limit inside each job. |
|
||||
| `--with-ablation`, `--ablation-out-dir PATH` | Schedule both suites together; optionally select a separate ablation root. |
|
||||
| `--pimsim-time-ms N` | Throughput convergence deadline; default: 1000 ms. |
|
||||
| `--no-fast` | Use full-duration throughput simulation for authoritative experiments. |
|
||||
| `--timeout-seconds N` | Per-stage timeout; default: 0 (unlimited). |
|
||||
@@ -45,11 +52,19 @@ Artifacts use
|
||||
under `artifacts/common/`
|
||||
and `comparison_report.{md,json}` under each comparison's `pimcomp/`.
|
||||
YOLO and GoogLeNet use `artifacts/common-full-model/` to avoid reusing historical Softmax-free references.
|
||||
References are prepared once per model. Each case starts when its own prerequisites finish.
|
||||
Pimcomp runs once per model/architecture/mode; later pipelines reuse its newly
|
||||
written report, including failures without simulation artifacts. Missing or
|
||||
unchanged anchor reports trigger fresh attempts. The runner continues after failures and exits nonzero if a
|
||||
unchanged prerequisite reports skip dependent cases without repeating Pimcomp work.
|
||||
Skipped cases retain unavailable metrics rather than old measurements.
|
||||
The runner continues after failures and exits nonzero if a
|
||||
compiler, simulation, or functional comparison fails.
|
||||
|
||||
Combined runs share references with ablation and reuse matching Raptor baseline reports.
|
||||
`ablation/baseline/source_reports.json` records those report locations.
|
||||
Only the parent process writes aggregate CSVs. Per-case logs and execution settings remain beside each case.
|
||||
Thread limits are recorded in reports and checked during report reuse.
|
||||
|
||||
If fast-mode convergence fails, the reports and CSV retain valid full-duration
|
||||
fallback throughput and output interval values. This is not a converged steady-state result.
|
||||
Per-case JSON records `throughput_measurement: full_duration_fallback` and
|
||||
|
||||
@@ -46,6 +46,7 @@ from raptor_validation.onnx_utils import (
|
||||
)
|
||||
from raptor_validation.raptor import compile_with_raptor, read_selected_pipeline_depth
|
||||
from raptor_validation.output_comparison import VALIDATION_POLICY, compare_output_arrays
|
||||
from raptor_validation.experiment_scheduler import available_processors, positive_integer, thread_environment
|
||||
from raptor_validation.pimsim_nn import (
|
||||
export_raptor_pimsim_artifact,
|
||||
parse_pimsim_nn_metrics,
|
||||
@@ -156,6 +157,7 @@ def run_logged(
|
||||
steps: list[StepRecord],
|
||||
stage: str | None = None,
|
||||
print_header: bool = True,
|
||||
env: dict[str, str] | None = None,
|
||||
) -> str:
|
||||
if print_header:
|
||||
print_step(name, cmd, cwd, stage)
|
||||
@@ -172,6 +174,7 @@ def run_logged(
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True,
|
||||
timeout=None if timeout_sec == 0 else timeout_sec,
|
||||
env=env,
|
||||
)
|
||||
except subprocess.TimeoutExpired as exc:
|
||||
duration = time.perf_counter() - start
|
||||
@@ -364,7 +367,8 @@ def compile_reference(
|
||||
run_logged(
|
||||
"Compile reference ONNX IR",
|
||||
[str(args.raptor_path), str(model_path), "-o", str(onnx_ir_base), "--EmitONNXIR",
|
||||
"--mlir-elide-elementsattrs-if-larger=16", "--enable-conv-opt-pass=false"],
|
||||
"--mlir-elide-elementsattrs-if-larger=16", "--enable-conv-opt-pass=false",
|
||||
f"-j={args.threads_per_job}"],
|
||||
cwd=REPO,
|
||||
timeout_sec=args.timeout_seconds,
|
||||
steps=steps,
|
||||
@@ -372,7 +376,7 @@ def compile_reference(
|
||||
)
|
||||
run_logged(
|
||||
"Compile reference native",
|
||||
[str(args.raptor_path), "-O3", str(model_path), "-o", str(runner_base)],
|
||||
[str(args.raptor_path), "-O3", str(model_path), "-o", str(runner_base), f"-j={args.threads_per_job}"],
|
||||
cwd=REPO,
|
||||
timeout_sec=args.timeout_seconds,
|
||||
steps=steps,
|
||||
@@ -400,7 +404,7 @@ def compile_reference(
|
||||
)
|
||||
run_logged(
|
||||
"Build runner",
|
||||
["cmake", "--build", ".", "-j"],
|
||||
["cmake", "--build", ".", "--parallel", str(args.threads_per_job)],
|
||||
cwd=build_dir,
|
||||
timeout_sec=args.timeout_seconds,
|
||||
steps=steps,
|
||||
@@ -446,6 +450,7 @@ def generate_reference_outputs(
|
||||
timeout_sec=args.timeout_seconds,
|
||||
steps=steps,
|
||||
print_header=print_header,
|
||||
env=thread_environment(1),
|
||||
)
|
||||
return reference_dir
|
||||
|
||||
@@ -576,6 +581,7 @@ def compile_raptor_target(
|
||||
f"--core-count={hardware['core_count']}",
|
||||
f"--pim-target-config={args.pimcomp_config}",
|
||||
"--pim-emit-json",
|
||||
f"-j={args.threads_per_job}",
|
||||
*args.raptor_extra_arg,
|
||||
]
|
||||
print_step("Compile Raptor", cmd, REPO, "Compile Pim")
|
||||
@@ -584,6 +590,7 @@ def compile_raptor_target(
|
||||
raptor_extra_args = [
|
||||
f"--pim-target-config={args.pimcomp_config}",
|
||||
"--pim-emit-json",
|
||||
f"-j={args.threads_per_job}",
|
||||
*args.raptor_extra_arg,
|
||||
]
|
||||
try:
|
||||
@@ -1202,6 +1209,7 @@ def experiment_context(args, model, functional_model, hardware, side):
|
||||
"oracle": args.pimsim_nn_build_dir / "ChipTest",
|
||||
"comparison": Path(__file__),
|
||||
"validation": VALIDATION_DIR / "raptor_validation/output_comparison.py",
|
||||
"thread_policy": VALIDATION_DIR / "raptor_validation/experiment_scheduler.py",
|
||||
"functional_simulator": args.pim_simulator_dir / "target/release/pim-simulator"}
|
||||
if side == "raptor":
|
||||
files["compiler"] = args.raptor_path
|
||||
@@ -1216,6 +1224,7 @@ def experiment_context(args, model, functional_model, hardware, side):
|
||||
settings.update({name: getattr(args, name) for name in (
|
||||
("raptor_extra_arg",) if side == "raptor" else
|
||||
("pimcomp_pipeline", "pimcomp_replication", "pimcomp_model_name"))})
|
||||
settings["threads_per_job"] = getattr(args, "threads_per_job", available_processors())
|
||||
return {"version": 2, "validation_policy": VALIDATION_POLICY,
|
||||
"files": fingerprint_files(files), "hardware": hardware, "settings": settings}
|
||||
|
||||
@@ -1569,9 +1578,9 @@ def main():
|
||||
parser.add_argument(
|
||||
"-j",
|
||||
"--jobs",
|
||||
type=int,
|
||||
default=4,
|
||||
help="Maximum parallel native reference runner processes (default: 4).",
|
||||
type=positive_integer,
|
||||
default=available_processors(),
|
||||
help="Maximum parallel native reference runner processes (default: logical processors).",
|
||||
)
|
||||
parser.add_argument("--pimcomp-pipeline", choices=["element", "batch"])
|
||||
parser.add_argument("--pimcomp-model-name", help="Use a Pimcomp built-in model name such as vgg16.")
|
||||
@@ -1613,7 +1622,10 @@ def main():
|
||||
action="store_true",
|
||||
help="Return a non-zero status if a stage or semantic validation fails.",
|
||||
)
|
||||
parser.add_argument("--threads-per-job", type=positive_integer, default=available_processors(),
|
||||
help="Threads for OpenMP, MLIR compilation, and simulation (default: logical processors).")
|
||||
args = parser.parse_args()
|
||||
os.environ.update(thread_environment(args.threads_per_job))
|
||||
if args.raptor_only and any(
|
||||
option is not None
|
||||
for option in (
|
||||
@@ -2152,6 +2164,7 @@ def main():
|
||||
"hardware": hardware,
|
||||
"pimsim_mode": args.pimsim_mode,
|
||||
"pimsim_time_ms": args.pimsim_time_ms,
|
||||
"threads_per_job": args.threads_per_job,
|
||||
"pimcomp_pipeline": args.pimcomp_pipeline,
|
||||
"pimcomp_replication": args.pimcomp_replication,
|
||||
"pimcomp_model_source": "not_run" if args.raptor_only else "original_onnx",
|
||||
|
||||
@@ -5,16 +5,12 @@ import argparse
|
||||
import csv
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import shlex
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from concurrent.futures import ProcessPoolExecutor, as_completed
|
||||
from contextlib import nullcontext, redirect_stderr, redirect_stdout
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
|
||||
from colorama import Fore, Style
|
||||
|
||||
@@ -31,7 +27,9 @@ from raptor_validation.artifacts import artifacts_dir, remove_lock_files
|
||||
from raptor_validation.pimsim_nn import parse_pimsim_nn_metrics
|
||||
from raptor_validation.output_comparison import VALIDATION_POLICY
|
||||
from raptor_validation.raptor import parse_pipeline_depth, pipeline_sort_key
|
||||
from raptor_validation.validate_one import STAGE_COLORS
|
||||
from raptor_validation.experiment_scheduler import (
|
||||
ExperimentTask, TaskResult, available_processors, positive_integer, run_tasks,
|
||||
)
|
||||
|
||||
PIMCOMP_SOURCE = REPO / "third_party/PIMCOMP-NN"
|
||||
PIMCOMP_CONFIGS = REPO / "validation/pimsim_configs/pimcomp"
|
||||
@@ -130,6 +128,8 @@ def write_results_csv(
|
||||
models: list[str],
|
||||
comparisons: tuple[tuple[str, int, str], ...] = COMPARISONS,
|
||||
ablation_variant: str | None = None,
|
||||
report_paths: dict[Path, Path | None] | None = None,
|
||||
raptor_only: bool = False,
|
||||
) -> Path:
|
||||
output = DEFAULT_RESULTS_PATH if root is None else root / RESULTS_FILENAME
|
||||
if ablation_variant is not None:
|
||||
@@ -179,6 +179,9 @@ def write_results_csv(
|
||||
report_path = result_dir(
|
||||
root, name, arch, comparison_mode, pipeline, ablation_variant
|
||||
) / "pimcomp/comparison_report.json"
|
||||
if report_paths is not None:
|
||||
directory = result_dir(root, name, arch, comparison_mode, pipeline, ablation_variant)
|
||||
report_path = report_paths.get(directory, report_path)
|
||||
row = {
|
||||
"model": name,
|
||||
"arch": arch,
|
||||
@@ -198,12 +201,12 @@ def write_results_csv(
|
||||
"better_compiler": "NA",
|
||||
"speedup": "NA",
|
||||
}
|
||||
if report_path.exists():
|
||||
if report_path is not None and report_path.exists():
|
||||
report = json.loads(report_path.read_text(encoding="utf-8"))
|
||||
if report.get("validation_policy") != VALIDATION_POLICY:
|
||||
raise ValueError(f"Report uses a different validation policy: {report_path}. Regenerate this comparison.")
|
||||
raptor_values = performance_values(report.get("raptor_performance") or {})
|
||||
pimcomp_values = performance_values(report.get("pimcomp_performance") or {})
|
||||
pimcomp_values = performance_values({} if raptor_only else report.get("pimcomp_performance") or {})
|
||||
raptor_metric = raptor_values["throughput"] if comparison_mode == "throughput" else raptor_values["latency"]
|
||||
pimcomp_metric = pimcomp_values["throughput"] if comparison_mode == "throughput" else pimcomp_values["latency"]
|
||||
row.update(
|
||||
@@ -211,7 +214,7 @@ def write_results_csv(
|
||||
report.get("raptor_validation")
|
||||
),
|
||||
pimcomp_functional_validation=functional_validation_status(
|
||||
report.get("pimcomp_validation")
|
||||
{"status": "skipped"} if raptor_only else report.get("pimcomp_validation")
|
||||
),
|
||||
raptor_throughput_samples_s=format_value(raptor_values["throughput"]),
|
||||
pimcomp_throughput_samples_s=format_value(pimcomp_values["throughput"]),
|
||||
@@ -322,17 +325,6 @@ def format_value(value: float | None) -> str:
|
||||
return f"{value:.6g}"
|
||||
|
||||
|
||||
def print_stage(title: str, color: str) -> None:
|
||||
print(Style.BRIGHT + color + f"[{title}]" + Style.RESET_ALL, flush=True)
|
||||
|
||||
|
||||
def print_completed(label: str, output: str = "") -> None:
|
||||
print(Style.BRIGHT + Fore.CYAN + f"[Completed {label}]" + Style.RESET_ALL)
|
||||
if output:
|
||||
print(output, end="" if output.endswith("\n") else "\n")
|
||||
print("=" * 72, flush=True)
|
||||
|
||||
|
||||
def run(command: list[str], *, dry_run: bool, check: bool = True) -> int:
|
||||
print(f" cwd: {REPO}", flush=True)
|
||||
print(f" $ {shlex.join(command)}", flush=True)
|
||||
@@ -341,29 +333,6 @@ def run(command: list[str], *, dry_run: bool, check: bool = True) -> int:
|
||||
return subprocess.run(command, cwd=REPO, check=check).returncode
|
||||
|
||||
|
||||
def run_comparison_job(job: tuple[str, list[str], Path | None]) -> tuple[str, int, str | None]:
|
||||
label, command, log_path = job
|
||||
sys.stdout.flush()
|
||||
sys.stderr.flush()
|
||||
if log_path is None:
|
||||
return label, run(command, dry_run=False, check=False), None
|
||||
|
||||
saved_stdout = os.dup(1)
|
||||
saved_stderr = os.dup(2)
|
||||
try:
|
||||
with open(log_path, "w", encoding="utf-8", buffering=1) as log:
|
||||
os.dup2(log.fileno(), 1)
|
||||
os.dup2(log.fileno(), 2)
|
||||
with redirect_stdout(log), redirect_stderr(log):
|
||||
returncode = run(command, dry_run=False, check=False)
|
||||
finally:
|
||||
os.dup2(saved_stdout, 1)
|
||||
os.dup2(saved_stderr, 2)
|
||||
os.close(saved_stdout)
|
||||
os.close(saved_stderr)
|
||||
return label, returncode, log_path.read_text(encoding="utf-8", errors="replace")
|
||||
|
||||
|
||||
def validate_pimcomp_source() -> None:
|
||||
header = PIMCOMP_SOURCE / "backend/GeneticAlgorithm.h"
|
||||
source = header.read_text(encoding="utf-8")
|
||||
@@ -469,7 +438,7 @@ def comparison_command_for(
|
||||
reuse_pimcomp_report = report
|
||||
elif reuse_shared_pimcomp:
|
||||
reuse_pimcomp_report = spec.shared_pimcomp_dir.parent / "comparison_report.json"
|
||||
return comparison_command(
|
||||
command = comparison_command(
|
||||
spec.model,
|
||||
spec.functional_model,
|
||||
spec.output_dir,
|
||||
@@ -491,6 +460,7 @@ def comparison_command_for(
|
||||
),
|
||||
reuse_pimcomp_report=reuse_pimcomp_report,
|
||||
)
|
||||
return command + ["--jobs", "1", "--threads-per-job", str(args.threads_per_job)]
|
||||
|
||||
|
||||
def pimcomp_report_stamp(output_dir: Path) -> tuple[int, int, int] | None:
|
||||
@@ -506,40 +476,6 @@ def pimcomp_report_ready(output_dir: Path, previous: tuple[int, int, int] | None
|
||||
return stamp is not None and stamp != previous
|
||||
|
||||
|
||||
def run_comparison_jobs(
|
||||
comparison_jobs: list[tuple[str, list[str], Path | None]],
|
||||
max_workers: int,
|
||||
log_dir: Path | None,
|
||||
log_offset: int,
|
||||
) -> tuple[list[str], int]:
|
||||
if not comparison_jobs:
|
||||
return [], log_offset
|
||||
print_directly = min(max_workers, len(comparison_jobs)) == 1
|
||||
jobs = [
|
||||
(
|
||||
label,
|
||||
command,
|
||||
None if print_directly or log_dir is None else log_dir / f"{log_offset + index}.log",
|
||||
)
|
||||
for index, (label, command, _) in enumerate(comparison_jobs)
|
||||
]
|
||||
with (nullcontext(None) if print_directly else ProcessPoolExecutor(max_workers=max_workers)) as executor:
|
||||
completed = (
|
||||
map(run_comparison_job, jobs)
|
||||
if print_directly
|
||||
else (
|
||||
future.result()
|
||||
for future in as_completed(executor.submit(run_comparison_job, job) for job in jobs)
|
||||
)
|
||||
)
|
||||
failed = []
|
||||
for label, returncode, output in completed:
|
||||
print_completed(label, output or "")
|
||||
if returncode:
|
||||
failed.append(label)
|
||||
return failed, log_offset + len(jobs)
|
||||
|
||||
|
||||
def ensure_throughput_config(arch: str, sim_time_ms: int) -> Path:
|
||||
source = PIMCOMP_CONFIGS / arch / "throughput_config_1000ms.json"
|
||||
target = PIMCOMP_CONFIGS / arch / f"throughput_config_{sim_time_ms}ms.json"
|
||||
@@ -566,7 +502,7 @@ def config_path(arch: str, mode: str, sim_time_ms: int, *, write: bool) -> Path:
|
||||
return path
|
||||
|
||||
|
||||
def main() -> int:
|
||||
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Compare supported Pimcomp models with Raptor latency and throughput schedules."
|
||||
)
|
||||
@@ -635,9 +571,9 @@ def main() -> int:
|
||||
parser.add_argument(
|
||||
"-j",
|
||||
"--jobs",
|
||||
type=int,
|
||||
default=4,
|
||||
help="Number of comparisons to run in parallel (default: 4).",
|
||||
type=positive_integer,
|
||||
default=available_processors(),
|
||||
help="Maximum concurrent comparisons (default: available logical processors).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--clean",
|
||||
@@ -657,7 +593,11 @@ def main() -> int:
|
||||
dest="raptor_extra_args",
|
||||
help="Additional argument to pass to Raptor; repeat as needed.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
parser.add_argument("--threads-per-job", type=positive_integer, default=available_processors(),
|
||||
help="Threads per compiler/simulator job (default: available logical processors).")
|
||||
parser.add_argument("--with-ablation", action="store_true", help="Run comparison and ablation in one dependency queue.")
|
||||
parser.add_argument("--ablation-out-dir", type=Path, help="Output root for a combined ablation run.")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
if args.ablation_variant and Path(args.ablation_variant).name != args.ablation_variant:
|
||||
parser.error("--ablation-variant must be a single directory name")
|
||||
@@ -665,25 +605,35 @@ def main() -> int:
|
||||
if args.only is not None and args.raptor_only:
|
||||
parser.error("--only cannot be combined with --raptor-only")
|
||||
|
||||
return args
|
||||
|
||||
|
||||
@dataclass
|
||||
class SuitePlan:
|
||||
args: argparse.Namespace
|
||||
root: Path | None
|
||||
common_root: Path | None
|
||||
comparisons: dict
|
||||
reference_batch_size: int
|
||||
specs: list[ComparisonSpec]
|
||||
aliases: dict[Path, Path]
|
||||
previous_reports: dict[Path, tuple[int, int, int] | None]
|
||||
|
||||
|
||||
def make_plan(args: argparse.Namespace) -> SuitePlan:
|
||||
out_dir = args.out_dir.resolve() if args.out_dir is not None else None
|
||||
common_root = args.common_dir.resolve() if args.common_dir is not None else None
|
||||
args.archs = list(dict.fromkeys(args.archs))
|
||||
args.models = list(dict.fromkeys(args.models))
|
||||
args.mode = list(dict.fromkeys(args.mode))
|
||||
if args.clean:
|
||||
print(
|
||||
f"Removed {clean_artifacts(out_dir, args.models, common_root)} "
|
||||
"comparison artifact path(s)."
|
||||
)
|
||||
return 0
|
||||
|
||||
if args.jobs < 1:
|
||||
parser.error("--jobs must be at least 1")
|
||||
raise ValueError("--jobs must be at least 1")
|
||||
if args.pimsim_time_ms <= 0:
|
||||
parser.error("--pimsim-time-ms must be positive")
|
||||
raise ValueError("--pimsim-time-ms must be positive")
|
||||
if args.batch_size <= 0:
|
||||
parser.error("--batch-size must be positive")
|
||||
raise ValueError("--batch-size must be positive")
|
||||
if args.timeout_seconds < 0:
|
||||
parser.error("--timeout-seconds must be non-negative")
|
||||
raise ValueError("--timeout-seconds must be non-negative")
|
||||
comparisons_by_arch: dict[str, tuple[tuple[str, int | str, str], ...]] = {}
|
||||
configs_by_arch: dict[str, dict[str, Path]] = {}
|
||||
for arch in args.archs:
|
||||
@@ -697,7 +647,7 @@ def main() -> int:
|
||||
and (args.pipeline is None or comparison[1] == args.pipeline)
|
||||
)
|
||||
if not comparisons:
|
||||
parser.error("--pipeline does not belong to the selected --mode")
|
||||
raise ValueError("--pipeline does not belong to the selected --mode")
|
||||
configs = {
|
||||
mode: config_path(arch, mode, args.pimsim_time_ms, write=not args.dry_run)
|
||||
for mode, _, _ in comparisons
|
||||
@@ -716,7 +666,7 @@ def main() -> int:
|
||||
if not MODELS[name].exists()
|
||||
]
|
||||
if missing:
|
||||
parser.error(f"missing model(s): {', '.join(missing)}")
|
||||
raise ValueError(f"missing model(s): {', '.join(missing)}")
|
||||
|
||||
if args.only is not None:
|
||||
missing_reuse = []
|
||||
@@ -730,7 +680,7 @@ def main() -> int:
|
||||
if not required.exists():
|
||||
missing_reuse.append(str(required))
|
||||
if missing_reuse:
|
||||
parser.error(
|
||||
raise ValueError(
|
||||
f"--only {args.only} needs existing comparison artifacts: "
|
||||
+ ", ".join(missing_reuse)
|
||||
)
|
||||
@@ -740,41 +690,6 @@ def main() -> int:
|
||||
if out_dir is not None and not args.dry_run:
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
print(Style.BRIGHT + f"Found {len(args.models)} Pimcomp model(s) to compare." + Style.RESET_ALL)
|
||||
print(f"Architectures: {', '.join(args.archs)}")
|
||||
print(f"Modes: {', '.join(args.mode)}")
|
||||
print(f"Throughput Pimsim time: {args.pimsim_time_ms} ms")
|
||||
print(f"Max parallel jobs: {args.jobs}")
|
||||
print(f"Shared reference batch: {reference_batch_size}")
|
||||
print(
|
||||
f"Comparison jobs: "
|
||||
f"{sum(len(args.models) * len(comparisons) for comparisons in comparisons_by_arch.values())}"
|
||||
)
|
||||
print(f"Artifacts root: {out_dir or SUITE}")
|
||||
print("=" * 72)
|
||||
|
||||
print_stage("Prepare shared artifacts", STAGE_COLORS["Build runner"])
|
||||
try:
|
||||
for name in args.models:
|
||||
try:
|
||||
run(
|
||||
prepare_common_command(
|
||||
MODELS[name],
|
||||
common_dir(out_dir, name, common_root),
|
||||
args.timeout_seconds,
|
||||
reference_batch_size,
|
||||
args.jobs,
|
||||
),
|
||||
dry_run=args.dry_run,
|
||||
)
|
||||
finally:
|
||||
print_completed(f"shared artifacts: {name}")
|
||||
except KeyboardInterrupt:
|
||||
remove_lock_files(out_dir or SUITE)
|
||||
print("Interrupted; cleaned validation lock files.", file=sys.stderr)
|
||||
return 130
|
||||
|
||||
failed = []
|
||||
comparison_specs: list[ComparisonSpec] = []
|
||||
shared_pimcomp_by_group: dict[tuple[str, str, str], Path] = {}
|
||||
for arch in args.archs:
|
||||
@@ -808,117 +723,171 @@ def main() -> int:
|
||||
)
|
||||
)
|
||||
|
||||
if args.dry_run:
|
||||
for spec in comparison_specs:
|
||||
command = comparison_command_for(
|
||||
spec,
|
||||
args,
|
||||
reuse_shared_pimcomp=not spec.anchor,
|
||||
)
|
||||
if run(command, dry_run=True, check=False):
|
||||
return SuitePlan(args, out_dir, common_root, comparisons_by_arch,
|
||||
reference_batch_size, comparison_specs, {}, {})
|
||||
|
||||
|
||||
def execution_key(plan: SuitePlan, spec: ComparisonSpec) -> tuple:
|
||||
args = plan.args
|
||||
return (spec.model, spec.common_dir, spec.config, spec.mode, spec.pipeline,
|
||||
tuple(args.raptor_extra_args), args.batch_size, args.no_fast,
|
||||
args.threads_per_job)
|
||||
|
||||
|
||||
def build_tasks(plans: list[SuitePlan]) -> list[ExperimentTask]:
|
||||
tasks = []
|
||||
references = {}
|
||||
comparison_sources = {execution_key(plan, spec): spec.output_dir
|
||||
for plan in plans if not plan.args.ablation_variant and plan.args.only is None
|
||||
and not plan.args.raptor_only for spec in plan.specs}
|
||||
for plan in plans:
|
||||
for name in plan.args.models:
|
||||
common = common_dir(plan.root, name, plan.common_root)
|
||||
previous = references.get(common)
|
||||
if previous is None or previous[0] < plan.reference_batch_size:
|
||||
references[common] = (plan.reference_batch_size, name, plan.args)
|
||||
for common, (batch_size, name, args) in references.items():
|
||||
command = prepare_common_command(MODELS[name], common, args.timeout_seconds,
|
||||
batch_size, args.threads_per_job)
|
||||
command += ["--threads-per-job", str(args.threads_per_job)]
|
||||
tasks.append(ExperimentTask("reference:" + str(common), lambda command=command: command,
|
||||
(), common / "prepare.log"))
|
||||
for plan in plans:
|
||||
args = plan.args
|
||||
for spec in plan.specs:
|
||||
source = comparison_sources.get(execution_key(plan, spec)) if args.ablation_variant == "baseline" else None
|
||||
report_root = source if source is not None else spec.output_dir
|
||||
plan.previous_reports[spec.output_dir] = pimcomp_report_stamp(report_root / "pimcomp/output")
|
||||
if args.ablation_variant == "baseline" and execution_key(plan, spec) in comparison_sources:
|
||||
plan.aliases[spec.output_dir] = comparison_sources[execution_key(plan, spec)]
|
||||
continue
|
||||
reuse = not (spec.anchor or args.only == "raptor" or args.raptor_only)
|
||||
dependencies = ["reference:" + str(spec.common_dir)]
|
||||
previous_report = pimcomp_report_stamp(spec.shared_pimcomp_dir) if reuse else None
|
||||
if reuse:
|
||||
dependencies.append(str(spec.shared_pimcomp_dir.parent.parent))
|
||||
|
||||
def command(spec=spec, args=args, reuse=reuse, previous=previous_report):
|
||||
if reuse and not args.dry_run and not pimcomp_report_ready(spec.shared_pimcomp_dir, previous):
|
||||
raise ValueError(f"Pimcomp prerequisite produced no fresh report: {spec.shared_pimcomp_dir}")
|
||||
return comparison_command_for(spec, args, reuse_shared_pimcomp=reuse)
|
||||
|
||||
tasks.append(ExperimentTask(str(spec.output_dir), command, tuple(dependencies),
|
||||
spec.output_dir / "comparison.log"))
|
||||
return tasks
|
||||
|
||||
|
||||
def report_for(plan: SuitePlan, spec: ComparisonSpec) -> Path:
|
||||
return plan.aliases.get(spec.output_dir, spec.output_dir) / "pimcomp/comparison_report.json"
|
||||
|
||||
|
||||
def finish_plan(plan: SuitePlan, results: dict[str, TaskResult]) -> int:
|
||||
args = plan.args
|
||||
failed, diagnostic = [], []
|
||||
overrides = {}
|
||||
for spec in plan.specs:
|
||||
directory = plan.aliases.get(spec.output_dir, spec.output_dir)
|
||||
result = results.get(str(directory))
|
||||
fresh = pimcomp_report_ready(directory / "pimcomp/output", plan.previous_reports[spec.output_dir])
|
||||
overrides[spec.output_dir] = report_for(plan, spec) if result and not result.returncode and fresh else None
|
||||
for arch in args.archs:
|
||||
output = write_results_csv(plan.root, arch, args.models, plan.comparisons[arch],
|
||||
args.ablation_variant, overrides, args.raptor_only)
|
||||
if plan.aliases:
|
||||
destination = (plan.root or SUITE) / "ablation" / args.ablation_variant / "source_reports.json"
|
||||
destination.write_text(json.dumps({str(k): str(v) if v is not None else None
|
||||
for k, v in overrides.items()}, indent=2) + "\n")
|
||||
for spec in plan.specs:
|
||||
result = results.get(str(plan.aliases.get(spec.output_dir, spec.output_dir)))
|
||||
path = report_for(plan, spec)
|
||||
report = json.loads(path.read_text()) if overrides[spec.output_dir] is not None else {}
|
||||
if result is None or result.returncode or overrides[spec.output_dir] is None:
|
||||
failed.append(spec.label)
|
||||
elif not comparison_passed(report):
|
||||
if args.ablation_variant == "no-sync" and completed_synchronization_ablation(report):
|
||||
diagnostic.append(spec.label)
|
||||
else:
|
||||
failed.append(spec.label)
|
||||
elif comparison_specs:
|
||||
raptor_only_run = args.only == "raptor" or args.raptor_only
|
||||
anchor_specs = comparison_specs if raptor_only_run else [
|
||||
spec for spec in comparison_specs if spec.anchor
|
||||
]
|
||||
anchor_jobs = [
|
||||
(
|
||||
spec.label,
|
||||
comparison_command_for(spec, args, reuse_shared_pimcomp=False),
|
||||
None,
|
||||
)
|
||||
for spec in anchor_specs
|
||||
]
|
||||
dependent_specs = [] if raptor_only_run else [
|
||||
spec for spec in comparison_specs if not spec.anchor
|
||||
]
|
||||
print_directly = min(args.jobs, len(comparison_specs)) == 1
|
||||
try:
|
||||
with (nullcontext(None) if print_directly else TemporaryDirectory(prefix="raptor-pimcomp-")) as log_dir:
|
||||
previous_reports = {
|
||||
spec.shared_pimcomp_dir: pimcomp_report_stamp(spec.shared_pimcomp_dir)
|
||||
for spec in anchor_specs
|
||||
}
|
||||
anchor_failed, log_offset = run_comparison_jobs(
|
||||
anchor_jobs,
|
||||
args.jobs,
|
||||
Path(log_dir) if log_dir else None,
|
||||
0,
|
||||
)
|
||||
failed.extend(anchor_failed)
|
||||
dependent_jobs = [
|
||||
(
|
||||
spec.label,
|
||||
comparison_command_for(
|
||||
spec,
|
||||
args,
|
||||
reuse_shared_pimcomp=pimcomp_report_ready(
|
||||
spec.shared_pimcomp_dir, previous_reports[spec.shared_pimcomp_dir]
|
||||
),
|
||||
),
|
||||
None,
|
||||
)
|
||||
for spec in dependent_specs
|
||||
]
|
||||
dependent_failed, _ = run_comparison_jobs(
|
||||
dependent_jobs,
|
||||
args.jobs,
|
||||
Path(log_dir) if log_dir else None,
|
||||
log_offset,
|
||||
)
|
||||
failed.extend(dependent_failed)
|
||||
except KeyboardInterrupt:
|
||||
remove_lock_files(out_dir or SUITE)
|
||||
print("Interrupted; cleaned validation lock files.", file=sys.stderr)
|
||||
return 130
|
||||
|
||||
if args.dry_run:
|
||||
return 1 if failed else 0
|
||||
|
||||
remove_lock_files(out_dir or SUITE)
|
||||
|
||||
results_path = None
|
||||
for arch in args.archs:
|
||||
results_path = write_results_csv(
|
||||
out_dir,
|
||||
arch,
|
||||
args.models,
|
||||
comparisons_by_arch[arch],
|
||||
args.ablation_variant,
|
||||
)
|
||||
assert results_path is not None
|
||||
print_stage(results_path.name, STAGE_COLORS["Compare outputs"])
|
||||
print(results_path.read_text(encoding="utf-8"), end="")
|
||||
diagnostic_failures = []
|
||||
for arch in args.archs:
|
||||
for name in args.models:
|
||||
for mode, pipeline, _ in comparisons_by_arch[arch]:
|
||||
label = f"{arch}/{name}/{mode}/pipeline{pipeline}"
|
||||
report_path = result_dir(
|
||||
out_dir, name, arch, mode, pipeline, args.ablation_variant
|
||||
) / "pimcomp/comparison_report.json"
|
||||
report = json.loads(report_path.read_text(encoding="utf-8")) if report_path.exists() else {}
|
||||
if not comparison_passed(report):
|
||||
if (label not in failed and args.ablation_variant == "no-sync"
|
||||
and completed_synchronization_ablation(report)):
|
||||
diagnostic_failures.append(label)
|
||||
elif label not in failed:
|
||||
failed.append(label)
|
||||
print(Style.BRIGHT + Fore.CYAN + "Summary" + Style.RESET_ALL)
|
||||
total_jobs = sum(len(args.models) * len(comparisons) for comparisons in comparisons_by_arch.values())
|
||||
print(Style.BRIGHT + f"Passed: {total_jobs - len(failed) - len(diagnostic_failures)}" + Style.RESET_ALL)
|
||||
print(Style.BRIGHT + f"Failed: {len(failed)}" + Style.RESET_ALL)
|
||||
if diagnostic_failures:
|
||||
print("No-sync numerical failures (diagnostic only): " + ", ".join(diagnostic_failures))
|
||||
print(Style.BRIGHT + f"Results: {results_path}" + Style.RESET_ALL)
|
||||
print(f"Results: {output}; passed: {len(plan.specs) - len(failed) - len(diagnostic)}; "
|
||||
f"failed: {len(failed)}; diagnostic no-sync failures: {len(diagnostic)}", flush=True)
|
||||
if failed:
|
||||
print(
|
||||
Fore.RED + f"Failed comparisons: {', '.join(failed)}" + Style.RESET_ALL,
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
return 0
|
||||
print("Failed comparisons: " + ", ".join(failed), file=sys.stderr)
|
||||
remove_lock_files(plan.root or SUITE)
|
||||
return int(bool(failed))
|
||||
|
||||
|
||||
def run_plans(plans: list[SuitePlan]) -> int:
|
||||
tasks = build_tasks(plans)
|
||||
args = plans[0].args
|
||||
if any((plan.args.jobs, plan.args.threads_per_job) != (args.jobs, args.threads_per_job) for plan in plans):
|
||||
raise ValueError("all experiment plans must use the same job and thread limits")
|
||||
if args.dry_run:
|
||||
for task in tasks:
|
||||
run(task.command(), dry_run=True, check=False)
|
||||
return 0
|
||||
print(f"Concurrent jobs: {args.jobs}; threads per job: {args.threads_per_job}", flush=True)
|
||||
results = run_tasks(tasks, args.jobs, args.threads_per_job, REPO)
|
||||
# Only the parent writes aggregate CSVs.
|
||||
failures = [finish_plan(plan, results) for plan in plans]
|
||||
return int(any(failures))
|
||||
|
||||
|
||||
def combined_ablation_plans(args: argparse.Namespace) -> tuple[list[SuitePlan], object, Path]:
|
||||
sys.path.insert(0, str(REPO / "validation/tools/pim/ablation"))
|
||||
import run_ablation as ablation
|
||||
from copy import deepcopy
|
||||
root = args.ablation_out_dir or args.out_dir or SUITE
|
||||
models = [name for name in args.models if name in ablation.ABLATION_DEFAULT_MODELS]
|
||||
if not models:
|
||||
raise ValueError("combined ablation needs at least one of the four ablation models")
|
||||
cases = [(mode, depth) for mode, depth, _ in COMPARISONS
|
||||
if mode in args.mode and (depth in (1, 4) if args.pipeline is None else depth == args.pipeline)]
|
||||
if args.pipeline is not None and not cases:
|
||||
cases = [("throughput", args.pipeline)]
|
||||
plans = []
|
||||
for variant in ablation.VARIANTS:
|
||||
for mode, depth in cases:
|
||||
if variant.name == "no-sync" and mode != "throughput":
|
||||
continue
|
||||
selected = deepcopy(args)
|
||||
selected.out_dir, selected.models = root, models
|
||||
selected.common_dir = args.common_dir or args.out_dir or SUITE
|
||||
selected.raptor_only, selected.only = True, None
|
||||
selected.ablation_variant = variant.name
|
||||
selected.mode, selected.pipeline = [mode], depth
|
||||
selected.raptor_extra_args += list(variant.raptor_args)
|
||||
plans.append(make_plan(selected))
|
||||
return plans, ablation, root
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = parse_args(argv)
|
||||
if args.clean:
|
||||
print(f"Removed {clean_artifacts(args.out_dir, args.models, args.common_dir)} comparison artifact path(s).")
|
||||
return 0
|
||||
try:
|
||||
plans = [make_plan(args)]
|
||||
if args.with_ablation:
|
||||
if args.only or args.raptor_only or args.ablation_variant:
|
||||
raise ValueError("--with-ablation requires a full compiler comparison")
|
||||
extra, ablation, ablation_root = combined_ablation_plans(args)
|
||||
plans.extend(extra)
|
||||
result = run_plans(plans)
|
||||
if args.with_ablation and not args.dry_run:
|
||||
cases = frozenset((spec.mode, str(spec.pipeline)) for plan in extra for spec in plan.specs)
|
||||
variants = [variant for variant in ablation.VARIANTS
|
||||
if any(plan.args.ablation_variant == variant.name for plan in extra)]
|
||||
_, failures = ablation.aggregate_results(ablation_root, variants, cases,
|
||||
frozenset(extra[0].args.models),
|
||||
output=ablation.DEFAULT_RESULTS_PATH if args.out_dir is None and args.ablation_out_dir is None else None)
|
||||
result = int(bool(result or failures))
|
||||
return result
|
||||
except KeyboardInterrupt:
|
||||
print("Interrupted; terminated experiment workers.", file=sys.stderr)
|
||||
return 130
|
||||
except ValueError as exc:
|
||||
print(str(exc), file=sys.stderr)
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user