diff --git a/README.md b/README.md index 94c88cb..4fa7769 100644 --- a/README.md +++ b/README.md @@ -73,6 +73,9 @@ differ, and validation defaults to Arch-A. ## Documentation +The [paper artifact](artifact/README.md) builds the pinned compiler and simulators +in Docker, runs the comparison and ablation experiments, and generates plots. + Start with the [documentation index](docs/README.md). It links the [compiler architecture](docs/compiler/architecture.md), [compiler options](docs/compiler/options.md), and diff --git a/artifact/.dockerignore b/artifact/.dockerignore new file mode 100644 index 0000000..2160ec4 --- /dev/null +++ b/artifact/.dockerignore @@ -0,0 +1,13 @@ +* +!Dockerfile +!pins.env +!requirements.lock +!models.sha256 +!build.sh +!run.py +!plot_ablation.py +!source.tar.gz +!source.sha256 +!validation-overlay/ +!validation-overlay/** +!validation-overlay.sha256 diff --git a/artifact/.gitignore b/artifact/.gitignore new file mode 100644 index 0000000..7afad21 --- /dev/null +++ b/artifact/.gitignore @@ -0,0 +1,7 @@ +/source.tar.gz +/source.tar.gz.tmp +/source.sha256 +/dist/ +/results/ +/validation-overlay/ +/validation-overlay.sha256 diff --git a/artifact/Dockerfile b/artifact/Dockerfile new file mode 100644 index 0000000..1d3b070 --- /dev/null +++ b/artifact/Dockerfile @@ -0,0 +1,46 @@ +FROM python:3.12.12-slim-trixie AS python-deps + +ENV DEBIAN_FRONTEND=noninteractive \ + CC=clang-19 CXX=clang++-19 \ + LDFLAGS=-fuse-ld=mold \ + RUSTFLAGS="-C linker=clang-19 -C link-arg=-fuse-ld=mold" \ + CARGO_HOME=/opt/cargo RUSTUP_HOME=/opt/rustup \ + PATH=/opt/raptor/.venv/bin:/opt/cargo/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ + OMP_NUM_THREADS=1 OPENBLAS_NUM_THREADS=1 MPLBACKEND=Agg \ + PYTHONDONTWRITEBYTECODE=1 PYTHONUNBUFFERED=1 \ + HOME=/tmp XDG_CACHE_HOME=/tmp/cache MPLCONFIGDIR=/tmp/matplotlib + +RUN apt-get update && apt-get install -y --no-install-recommends \ + build-essential ca-certificates clang-19 curl git libomp-19-dev \ + libzstd-dev mold ninja-build pkg-config zlib1g-dev \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /opt/artifact +COPY pins.env requirements.lock models.sha256 build.sh ./ +RUN python -m venv /opt/raptor/.venv \ + && /opt/raptor/.venv/bin/python -m pip install --no-cache-dir --only-binary=onnx -r requirements.lock + +FROM python-deps AS artifact +ARG BUILD_JOBS +RUN . ./pins.env && curl --proto '=https' --tlsv1.2 -fsSL https://sh.rustup.rs -o /tmp/rustup.sh \ + && sh /tmp/rustup.sh -y --profile minimal --default-toolchain "$RUST_VERSION" \ + && rm /tmp/rustup.sh + +COPY source.tar.gz source.sha256 ./ +RUN sha256sum -c source.sha256 \ + && . ./pins.env \ + && gzip -dc source.tar.gz > /tmp/raptor-source.tar \ + && test "$(git get-tar-commit-id < /tmp/raptor-source.tar)" = "$RAPTOR_COMMIT" \ + && tar -xf /tmp/raptor-source.tar -C /opt/raptor \ + && rm source.tar.gz /tmp/raptor-source.tar +RUN bash build.sh "${BUILD_JOBS:-$(env -u OMP_NUM_THREADS -u OMP_THREAD_LIMIT nproc)}" +COPY validation-overlay/ ./validation-overlay/ +COPY validation-overlay.sha256 ./ +RUN sha256sum -c validation-overlay.sha256 \ + && cp -R validation-overlay/validation/. /opt/raptor/validation/ \ + && cp validation-overlay.sha256 /opt/artifact/provenance/ +COPY run.py plot_ablation.py ./ + +WORKDIR /opt/raptor +ENTRYPOINT ["/opt/raptor/.venv/bin/python", "/opt/artifact/run.py"] +CMD ["reproduce"] diff --git a/artifact/README.md b/artifact/README.md new file mode 100644 index 0000000..978b1d9 --- /dev/null +++ b/artifact/README.md @@ -0,0 +1,32 @@ +# Raptor paper artifact + +Raptor is fixed to commit `4a1fe710b2f0e4e9c0504f4fd9b4c4dd3df53b7d`. + +Requires Docker on Linux or WSL2. The build downloads dependencies and uses Clang 19 and mold. +Install the host dependencies: + +```bash +sudo apt install -y docker.io docker-cli docker-buildx git +sudo systemctl enable --now docker +sudo usermod -aG docker "$USER" +newgrp docker +``` + +Log out and back in, then run from this directory: + +```bash +bash artifact.sh build +bash artifact.sh reproduce +``` + +Open `results/full/index.html` to view the plots and result tables. +The page also links to the CSV files and logs. Failed numerical validation remains visible in the results. + +For a small VGG8 run, use `bash artifact.sh smoke` and open `results/smoke/index.html`. +Build jobs, run jobs, and threads per job default to the available logical processor count. +Override with `BUILD_JOBS`, `JOBS`, and `THREADS_PER_JOB`, or run `bash artifact.sh reproduce --jobs 8 --threads-per-job 4`. +For another run, set `OUTPUT_DIR` to a new directory. + +To prepare the paper attachment, run `bash artifact/artifact.sh package` from the repository root. +The archive is written to `artifact/dist/` and includes the pinned sources and models. +Updated scheduling code is included separately; compiler and simulator sources stay pinned. diff --git a/artifact/artifact.sh b/artifact/artifact.sh new file mode 100755 index 0000000..257f0e5 --- /dev/null +++ b/artifact/artifact.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +set -euo pipefail +cd -- "$(dirname -- "${BASH_SOURCE[0]}")" +source pins.env +image="raptor-artifact:${RAPTOR_COMMIT:0:12}" +command=${1:-help} +if (($#)); then shift; fi +case "$command" in + prepare) + exec bash prepare.sh + ;; + package) + bash prepare.sh + mkdir -p dist + archive="raptor-artifact-${RAPTOR_COMMIT:0:12}.tar.gz" + package_dir=$(mktemp -d) + trap 'rm -rf -- "$package_dir"' EXIT + mkdir "$package_dir/raptor-artifact" + cp \ + README.md artifact.sh prepare.sh Dockerfile .dockerignore pins.env \ + requirements.lock models.sha256 build.sh run.py plot_ablation.py source.tar.gz source.sha256 \ + "$package_dir/raptor-artifact/" + cp -R validation-overlay "$package_dir/raptor-artifact/" + cp validation-overlay.sha256 "$package_dir/raptor-artifact/" + tar -czf "dist/$archive" -C "$package_dir" raptor-artifact + (cd dist && sha256sum "$archive" > "$archive.sha256") + printf 'Attachment: %s/dist/%s\n' "$PWD" "$archive" + ;; + build) + [[ -f source.tar.gz ]] || bash prepare.sh + if [[ -f ../validation/raptor_validation/experiment_scheduler.py ]]; then + bash prepare.sh --overlay-only + fi + sha256sum -c source.sha256 + docker build --build-arg "BUILD_JOBS=${BUILD_JOBS:-$(env -u OMP_NUM_THREADS -u OMP_THREAD_LIMIT nproc)}" -t "$image" . + ;; + reproduce|smoke|plots) + output=${OUTPUT_DIR:-"$PWD/results"} + mkdir -p "$output" + output=$(cd -- "$output" && pwd) + image_id=$(docker image inspect --format '{{.Id}}' "$image") + status=0 + docker run --rm --init --network none \ + --user "$(id -u):$(id -g)" \ + --env "ARTIFACT_IMAGE_ID=$image_id" \ + --env "JOBS=${JOBS:-$(env -u OMP_NUM_THREADS -u OMP_THREAD_LIMIT nproc)}" \ + --env "THREADS_PER_JOB=${THREADS_PER_JOB:-$(env -u OMP_NUM_THREADS -u OMP_THREAD_LIMIT nproc)}" \ + --mount "type=bind,src=$output,dst=/results" \ + "$image" "$command" "$@" || status=$? + profile=full + if [[ "$command" = smoke || " $* " = *' --smoke '* ]]; then profile=smoke; fi + if [[ -f "$output/$profile/index.html" ]]; then + printf '\nOpen %s/%s/index.html\n' "$output" "$profile" + fi + exit "$status" + ;; + *) + echo 'Usage: bash artifact.sh {build|smoke|reproduce|plots|package}' + [[ "$command" = help ]] + ;; +esac diff --git a/artifact/build.sh b/artifact/build.sh new file mode 100644 index 0000000..80c3535 --- /dev/null +++ b/artifact/build.sh @@ -0,0 +1,75 @@ +#!/usr/bin/env bash +set -euo pipefail +source /opt/artifact/pins.env +export CMAKE_BUILD_PARALLEL_LEVEL=${1:-$(env -u OMP_NUM_THREADS -u OMP_THREAD_LIMIT nproc)} +[[ "$CMAKE_BUILD_PARALLEL_LEVEL" =~ ^[1-9][0-9]*$ ]] +cd /opt/raptor +sha256sum -c /opt/artifact/models.sha256 + +checkout() { + local url=$1 revision=$2 destination=$3 + git init "$destination" + git -C "$destination" remote add origin "$url" + git -C "$destination" fetch --depth 1 origin "$revision" + git -C "$destination" checkout --detach FETCH_HEAD + test "$(git -C "$destination" rev-parse HEAD)" = "$revision" + git -C "$destination" submodule update --init --recursive --depth 1 +} + +checkout https://github.com/onnx/onnx-mlir.git "$ONNX_MLIR_COMMIT" onnx-mlir +checkout https://github.com/llvm/llvm-project.git "$LLVM_COMMIT" onnx-mlir/llvm-project +checkout https://github.com/HEAPLab/PIMCOMP-NN.git "$PIMCOMP_COMMIT" third_party/PIMCOMP-NN +checkout https://github.com/HEAPLab/pimsim-nn.git "$PIMSIM_COMMIT" backend-simulators/pim/pimsim-nn +checkout https://github.com/protocolbuffers/protobuf.git "$PROTOBUF_COMMIT" /opt/protobuf + +common=(-G Ninja -DCMAKE_BUILD_TYPE=Release -DCMAKE_POSITION_INDEPENDENT_CODE=ON + -DCMAKE_EXE_LINKER_FLAGS=-fuse-ld=mold + -DCMAKE_SHARED_LINKER_FLAGS=-fuse-ld=mold + -DCMAKE_MODULE_LINKER_FLAGS=-fuse-ld=mold) + +cmake -S /opt/protobuf -B /opt/protobuf/build "${common[@]}" \ + -Dprotobuf_BUILD_TESTS=OFF -Dprotobuf_ABSL_PROVIDER=module \ + -DABSL_PROPAGATE_CXX_STD=ON -DCMAKE_CXX_STANDARD=17 +cmake --build /opt/protobuf/build +cmake --install /opt/protobuf/build +ldconfig + +llvm=onnx-mlir/llvm-project +cmake -S "$llvm/llvm" -B "$llvm/build_release" "${common[@]}" \ + '-DLLVM_ENABLE_PROJECTS=mlir;clang' -DLLVM_ENABLE_RUNTIMES=openmp \ + -DLLVM_TARGETS_TO_BUILD=host -DLLVM_ENABLE_ASSERTIONS=ON -DLLVM_ENABLE_RTTI=ON \ + -DENABLE_LIBOMPTARGET=OFF -DLLVM_ENABLE_LIBEDIT=OFF \ + -DLLVM_PARALLEL_LINK_JOBS="$CMAKE_BUILD_PARALLEL_LEVEL" +cmake --build "$llvm/build_release" + +cmake -S . -B build_release "${common[@]}" \ + -DONNX_MLIR_ACCELERATORS=PIM -DLLVM_ENABLE_ASSERTIONS=ON \ + -DONNX_MLIR_ENABLE_JAVA=OFF -DONNX_MLIR_BUILD_TESTS=OFF \ + -DMLIR_DIR="$PWD/$llvm/build_release/lib/cmake/mlir" +cmake --build ./build_release + +cmake -S third_party/PIMCOMP-NN -B third_party/PIMCOMP-NN/build "${common[@]}" +cmake --build third_party/PIMCOMP-NN/build --target PIMCOMP-NN +sim=backend-simulators/pim/pimsim-nn +cmake -S "$sim" -B "$sim/build" "${common[@]}" +cmake --build "$sim/build" +ctest --test-dir "$sim/build" --output-on-failure + +crate=backend-simulators/pim/pim-simulator +cargo build --locked --release --no-default-features --manifest-path "$crate/Cargo.toml" \ + --package pim-simulator --bin pim-simulator --jobs "$CMAKE_BUILD_PARALLEL_LEVEL" +# The experiment driver uses cargo run. All dependencies must be available offline. +cargo fetch --locked --manifest-path "$crate/Cargo.toml" +mkdir -p /opt/artifact/provenance +cp /opt/artifact/pins.env /opt/artifact/source.sha256 /opt/artifact/models.sha256 /opt/artifact/provenance/ +for dir in onnx-mlir "$llvm" third_party/PIMCOMP-NN "$sim" /opt/protobuf; do + printf '%s %s\n' "$dir" "$(git -C "$dir" rev-parse HEAD)" + git -C "$dir" submodule status --recursive +done > /opt/artifact/provenance/revisions.txt +dpkg-query -W > /opt/artifact/provenance/system-packages.txt +.venv/bin/python -m pip freeze > /opt/artifact/provenance/python-packages.txt +{ clang-19 --version; mold --version; cmake --version; rustc --version; } \ + > /opt/artifact/provenance/toolchain.txt +# Runtime uses an arbitrary host UID. Cargo and the runners need writable caches +# and ONNX-MLIR writes temporary shims/configurations inside its source tree. +chmod -R a+rwX /opt/raptor /opt/cargo /opt/rustup diff --git a/artifact/models.sha256 b/artifact/models.sha256 new file mode 100644 index 0000000..9b4a30a --- /dev/null +++ b/artifact/models.sha256 @@ -0,0 +1,5 @@ +788088b908e233d924c7c26b997e89ee861290c7bc56783a306e8201d79aac8f validation/networks/pimcomp_models/resnet18/resnet18-v1-7.onnx +c3231061d081bdd47884137b02134f85142752a39e87263c529cd14ed242b096 validation/networks/pimcomp_models/resnet34/resnet34-v1-7.onnx +c960c195953423388733cd0e60712cf425ca1f8917e264f5cf98e028752c61cf validation/networks/pimcomp_models/googlenet/googlenet-12-no-lrn.onnx +396cdea21e5e7d02c3f26f14d22ef20975171702493f5c5e79b8e0d896e541ef validation/networks/pimcomp_models/vgg8/vgg8-mnist-reconstructed.onnx +5e7bd85dc60b252674f8d7e1ad24f2317d00dfb8c524826536698462de04620d validation/networks/pimcomp_models/yolo11n/yolo11n.onnx diff --git a/artifact/pins.env b/artifact/pins.env new file mode 100644 index 0000000..602db15 --- /dev/null +++ b/artifact/pins.env @@ -0,0 +1,8 @@ +# Raptor main at artifact preparation time. Do not replace this with a branch. +RAPTOR_COMMIT=4a1fe710b2f0e4e9c0504f4fd9b4c4dd3df53b7d +ONNX_MLIR_COMMIT=eb54c2afc46d00c6b196d1f275b6bfee17e12f69 +LLVM_COMMIT=0c2701fe7fa002e1befc5f86c268a7964f96d286 +PIMCOMP_COMMIT=7b6be6da4d33bf19f68d073eccac2a78e326a5af +PIMSIM_COMMIT=17e37c15da5473362e53268dabdd07dff5fe8fbf +PROTOBUF_COMMIT=6a6cd88c262ffdb1738167a47d5fcc7a3eb4edac +RUST_VERSION=1.90.0 diff --git a/artifact/plot_ablation.py b/artifact/plot_ablation.py new file mode 100644 index 0000000..9785486 --- /dev/null +++ b/artifact/plot_ablation.py @@ -0,0 +1,59 @@ +"""Plot planning and synchronization gains without hiding invalid programs.""" +from __future__ import annotations + +import argparse +import csv +import math +from pathlib import Path + +import matplotlib +matplotlib.use('Agg') +import matplotlib.pyplot as plt + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--input', type=Path, required=True) + parser.add_argument('--output-dir', type=Path, required=True) + args = parser.parse_args() + with args.input.open(newline='') as stream: + rows = list(csv.DictReader(stream)) + if not rows: + raise ValueError('The ablation CSV is empty') + panels = [('plan', 'latency', 'latency_gain_percent', 'Planning: latency reduction (%)'), + ('plan', 'throughput', 'throughput_gain_percent', 'Planning: throughput increase (%)'), + ('sync', 'throughput', 'throughput_gain_percent', 'Synchronization: throughput increase (%)')] + fig, axes = plt.subplots(3, 1, figsize=(11, 9), layout='constrained') + for ax, (comparison, mode, field, label) in zip(axes, panels): + selected = sorted((r for r in rows if r['comparison'] == comparison and r['mode'] == mode), + key=lambda r: (r['arch'], r['model'], int(r['raptor_pipeline']))) + for x, row in enumerate(selected): + try: + value = float(row[field].rstrip('%')) + except (ValueError, TypeError): + value = math.nan + valid = row['functional_validation'] == row['baseline_functional_validation'] == 'PASS' + if math.isfinite(value): + ax.bar(x, value, color='#B3D5CA' if valid else '#FF9494', + edgecolor='#222222', hatch='' if valid else '//') + ax.annotate(f'{value:.1f}' + ('' if valid else ' INVALID'), + (x, value), xytext=(0, 4 if value >= 0 else -4), + textcoords='offset points', ha='center', + va='bottom' if value >= 0 else 'top', fontsize=7) + else: + ax.text(x, 0, 'NA', ha='center', va='bottom', fontsize=8) + ax.set_xticks(range(len(selected)), + [f"{r['model']}\n{r['arch']} p{r['raptor_pipeline']}" for r in selected], fontsize=8) + ax.set_ylabel(label) + ax.axhline(0, color='#222222', linewidth=.7) + ax.set_xlim(-.6, max(.6, len(selected) - .4)) + ax.margins(y=.3) + fig.suptitle('Raptor ablation (red hatching: functional validation failed)') + args.output_dir.mkdir(parents=True, exist_ok=True) + for extension in ('pdf', 'svg', 'png'): + fig.savefig(args.output_dir / f'ablation.{extension}', dpi=200) + plt.close(fig) + + +if __name__ == '__main__': + main() diff --git a/artifact/prepare.sh b/artifact/prepare.sh new file mode 100644 index 0000000..5adbdb5 --- /dev/null +++ b/artifact/prepare.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +set -euo pipefail +cd -- "$(dirname -- "${BASH_SOURCE[0]}")" +source pins.env +repo=$(git -C .. rev-parse --show-toplevel) +test "$(git -C "$repo" rev-parse "$RAPTOR_COMMIT^{commit}")" = "$RAPTOR_COMMIT" +for item in "onnx-mlir:$ONNX_MLIR_COMMIT" \ + "third_party/PIMCOMP-NN:$PIMCOMP_COMMIT" \ + "backend-simulators/pim/pimsim-nn:$PIMSIM_COMMIT"; do + path=${item%:*} + expected=${item##*:} + actual=$(git -C "$repo" rev-parse "$RAPTOR_COMMIT:$path") + test "$actual" = "$expected" +done +if [[ "${1:-}" != --overlay-only ]]; then + # Include the complete compiler and evaluation code, plus only the suite models. + # The last two paths are the targets of the suite's relative model symlinks. + git -C "$repo" archive --format=tar "$RAPTOR_COMMIT" \ + CMakeLists.txt README.md .gitmodules src test docs backend-simulators \ + validation/README.md validation/validate.py validation/raptor_validation \ + validation/tools validation/pimsim_configs validation/networks/pimcomp_models \ + validation/networks/resnet18/depth_68/resnet18_depth_68.onnx \ + validation/networks/yolo11n/depth_51/yolo11n_depth_51.onnx \ + | gzip -n > source.tar.gz.tmp + mv source.tar.gz.tmp source.tar.gz + sha256sum source.tar.gz > source.sha256 +fi +overlay=(validation/raptor_validation/experiment_scheduler.py + validation/tools/pim/pimcomp/compare/run_pimcomp_models.py + validation/tools/pim/pimcomp/compare/compare_raptor_pimcomp_model.py + validation/tools/pim/ablation/run_ablation.py + validation/tools/pim/pimcomp/compare/README.md + validation/tools/pim/ablation/README.md) +for path in "${overlay[@]}"; do + mkdir -p "validation-overlay/$(dirname "$path")" + cp "$repo/$path" "validation-overlay/$path" +done +sha256sum "${overlay[@]/#/validation-overlay/}" > validation-overlay.sha256 +printf 'Prepared Raptor %s\n' "$RAPTOR_COMMIT" diff --git a/artifact/requirements.lock b/artifact/requirements.lock new file mode 100644 index 0000000..85e3f14 --- /dev/null +++ b/artifact/requirements.lock @@ -0,0 +1,23 @@ +# Evaluation and ONNX-MLIR build dependencies. The Graph Explorer is not required. +cmake==3.31.6 +colorama==0.4.6 +contourpy==1.3.2 +cycler==0.12.1 +execnet==2.1.1 +fonttools==4.58.0 +iniconfig==2.1.0 +kiwisolver==1.4.8 +lit==15.0.7 +matplotlib==3.10.3 +numpy==2.2.6 +onnx==1.17.0 +packaging==25.0 +pillow==11.2.1 +pluggy==1.5.0 +protobuf==7.34.0 +pyparsing==3.2.3 +pytest==8.3.5 +pytest-xdist==3.6.1 +python-dateutil==2.9.0.post0 +PyYAML==6.0.2 +six==1.17.0 diff --git a/artifact/run.py b/artifact/run.py new file mode 100644 index 0000000..cd73210 --- /dev/null +++ b/artifact/run.py @@ -0,0 +1,140 @@ +"""Container commands for the pinned Raptor paper artifact.""" +from __future__ import annotations + +import argparse +import csv +from datetime import datetime, timezone +from html import escape +import json +import os +from pathlib import Path +import platform +import shutil +import subprocess +import sys +import time + +HERE = Path(__file__).resolve().parent +REPO = Path('/opt/raptor') +PYTHON = REPO / '.venv/bin/python' +COMPARE = Path('validation/tools/pim/pimcomp/compare') +ABLATION = Path('validation/tools/pim/ablation/run_ablation.py') + + +def stage(name: str, command: list[str], root: Path) -> dict: + print(f'[{name}] {command}', flush=True) + start = time.monotonic() + with (root / f'{name}.log').open('wb') as log: + with subprocess.Popen(command, cwd=REPO, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) as process: + for line in process.stdout: + sys.stdout.buffer.write(line) + sys.stdout.buffer.flush() + log.write(line) + returncode = process.wait() + return {'command': command, 'exit_code': returncode, + 'seconds': round(time.monotonic() - start, 3)} + + +def plots(root: Path, smoke: bool) -> dict: + commands = {'ablation-plots': [str(PYTHON), str(HERE / 'plot_ablation.py'), + '--input', str(root / 'ablation/results_ablation.csv'), + '--output-dir', str(root / 'plots/ablation')]} + if not smoke: + commands['comparison-plots'] = [str(PYTHON), str(COMPARE / 'generate_plots.py'), + '--input', str(root / 'comparison/results_comparison.csv'), + '--output-dir', str(root / 'plots/comparison')] + return {name: stage(name, command, root) for name, command in commands.items()} + + +def show_results(root: Path) -> None: + page = ['', + '', + 'Raptor results

Raptor results

', + '

Functional validation status is shown in the tables and plots.

'] + for title, relative in ( + ('Comparison', 'plots/comparison/png/performance_energy_compact.png'), + ('Pipeline throughput', 'plots/comparison/png/raptor_pipeline_throughput.png'), + ('Ablation', 'plots/ablation/ablation.png'), + ): + if (root / relative).is_file(): + page.append(f'

{title}

{title}') + for title, relative in (('Comparison', 'comparison/results_comparison.csv'), + ('Ablation', 'ablation/results_ablation.csv')): + path = root / relative + if path.is_file(): + page.append(f'

{title} data

Download CSV

') + with path.open(newline='') as stream: + for index, row in enumerate(csv.reader(stream)): + tag = 'th' if index == 0 else 'td' + page.append('' + ''.join(f'<{tag}>{escape(value)}' for value in row) + '') + page.append('
') + page.append('

Files

') + (root / 'index.html').write_text('\n'.join(page) + '\n') + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('command', choices=('reproduce', 'smoke', 'plots')) + parser.add_argument('--smoke', action='store_true', help='Plot the smoke results.') + parser.add_argument('--jobs', type=int, default=int(os.environ.get('JOBS', len(os.sched_getaffinity(0))))) + parser.add_argument('--threads-per-job', type=int, + default=int(os.environ.get('THREADS_PER_JOB', len(os.sched_getaffinity(0))))) + args = parser.parse_args() + smoke = args.command == 'smoke' or args.smoke + root = Path('/results') / ('smoke' if smoke else 'full') + os.environ['CARGO_NET_OFFLINE'] = 'true' + if args.command == 'plots': + manifest_path = root / 'run.json' + manifest = json.loads(manifest_path.read_text()) + records = plots(root, smoke) + manifest['stages'].update(records) + manifest_path.write_text(json.dumps(manifest, indent=2) + '\n') + show_results(root) + return int(any(record['exit_code'] for record in records.values())) + if root.exists(): + parser.error(f'{root} exists. Select a fresh OUTPUT_DIR to prevent stale result reuse.') + jobs = args.jobs + if jobs < 1 or args.threads_per_job < 1: + parser.error('jobs and threads-per-job must be positive') + root.mkdir(parents=True) + shutil.copytree(HERE / 'provenance', root / 'provenance') + # Copy the artifact scripts so each result identifies its orchestration code. + shutil.copytree(HERE, root / 'provenance/artifact', + ignore=shutil.ignore_patterns('provenance', '__pycache__')) + manifest = {'started_utc': datetime.now(timezone.utc).isoformat(), + 'profile': 'smoke' if smoke else 'full', 'image_id': os.environ.get('ARTIFACT_IMAGE_ID'), + 'platform': platform.platform(), 'machine': platform.machine(), + 'cpu_count': os.cpu_count(), 'jobs': jobs, 'threads_per_job': args.threads_per_job, + 'batch_size': 64, 'fast': False, 'stages': {}} + common = ['--jobs', str(jobs), '--threads-per-job', str(args.threads_per_job), + '--batch-size', '64', '--no-fast'] + subset = ['--models', 'vgg8', '--archs', 'arch-a', '--mode', 'latency', '--pipeline', '1'] if smoke else [] + commands = { + 'experiments': [str(PYTHON), str(COMPARE / 'run_pimcomp_models.py'), + '--out-dir', str(root / 'comparison'), + '--with-ablation', '--ablation-out-dir', str(root / 'ablation'), + '--common-dir', str(root / 'references'), *common, *subset], + } + manifest_path = root / 'run.json' + manifest_path.write_text(json.dumps(manifest, indent=2) + '\n') + for name, command in commands.items(): + manifest['stages'][name] = stage(name, command, root) + manifest_path.write_text(json.dumps(manifest, indent=2) + '\n') + manifest['stages'].update(plots(root, smoke)) + manifest['finished_utc'] = datetime.now(timezone.utc).isoformat() + manifest_path.write_text(json.dumps(manifest, indent=2) + '\n') + show_results(root) + return int(any(record['exit_code'] for record in manifest['stages'].values())) + + +if __name__ == '__main__': + raise SystemExit(main()) diff --git a/validation/raptor_validation/experiment_scheduler.py b/validation/raptor_validation/experiment_scheduler.py new file mode 100644 index 0000000..6bfc411 --- /dev/null +++ b/validation/raptor_validation/experiment_scheduler.py @@ -0,0 +1,153 @@ +"""Run experiment subprocesses as their dependencies complete.""" +from __future__ import annotations + +import argparse +from collections import deque +from dataclasses import dataclass +import json +import os +from pathlib import Path +import signal +import subprocess +import time +from typing import Callable + + +def available_processors() -> int: + if hasattr(os, "sched_getaffinity"): + return len(os.sched_getaffinity(0)) + return os.cpu_count() or 1 + + +def positive_integer(value: str) -> int: + number = int(value) + if number < 1: + raise argparse.ArgumentTypeError("must be positive") + return number + + +def thread_environment(threads: int) -> dict[str, str]: + return {**os.environ, "OMP_NUM_THREADS": str(threads), + "OMP_THREAD_LIMIT": str(threads), "OMP_DYNAMIC": "FALSE", + "OMP_WAIT_POLICY": "PASSIVE", "OPENBLAS_NUM_THREADS": str(threads), + "MKL_NUM_THREADS": str(threads), "RAYON_NUM_THREADS": str(threads), + "CMAKE_BUILD_PARALLEL_LEVEL": str(threads), "CARGO_BUILD_JOBS": str(threads)} + + +@dataclass(frozen=True) +class ExperimentTask: + key: str + command: Callable[[], list[str]] + dependencies: tuple[str, ...] + log_path: Path + + +@dataclass(frozen=True) +class TaskResult: + returncode: int + error: str = "" + + +def stop_processes(processes: list[subprocess.Popen]) -> None: + for process in processes: + try: + os.killpg(process.pid, signal.SIGTERM) + except ProcessLookupError: + pass + deadline = time.monotonic() + 5 + for process in processes: + try: + process.wait(timeout=max(0, deadline - time.monotonic())) + except subprocess.TimeoutExpired: + try: + os.killpg(process.pid, signal.SIGKILL) + except ProcessLookupError: + pass + process.wait() + + +def run_tasks(tasks: list[ExperimentTask], jobs: int, threads: int, + cwd: Path) -> dict[str, TaskResult]: + if jobs < 1 or threads < 1: + raise ValueError("jobs and threads-per-job must be positive") + by_key = {task.key: task for task in tasks} + if len(by_key) != len(tasks): + raise ValueError("duplicate experiment task") + waiting = {task.key: set(task.dependencies) for task in tasks} + children = {key: [] for key in by_key} + for task in tasks: + for dependency in task.dependencies: + if dependency not in by_key: + raise ValueError(f"unknown dependency: {dependency}") + children[dependency].append(task.key) + ready = deque(key for key in by_key if not waiting[key]) + degrees = {key: len(dependencies) for key, dependencies in waiting.items()} + pending = deque(ready) + visited = 0 + while pending: + visited += 1 + for child in children[pending.popleft()]: + degrees[child] -= 1 + if not degrees[child]: + pending.append(child) + if visited != len(tasks): + raise ValueError("experiment dependency cycle") + active = {} + results = {} + records = {} + environment = thread_environment(threads) + + def complete(key: str, result: TaskResult) -> None: + results[key] = result + task = by_key[key] + record = records.get(key, {"dependencies": task.dependencies, "threads_per_job": threads}) + record.update(returncode=result.returncode, error=result.error) + task.log_path.parent.mkdir(parents=True, exist_ok=True) + task.log_path.with_suffix(".execution.json").write_text(json.dumps(record, indent=2) + "\n") + print(f"[{'Failed' if result.returncode else 'Completed'}] {key}" + f"{': ' + result.error if result.error else ''}", flush=True) + for child in children[key]: + waiting[child].remove(key) + if not waiting[child]: + ready.append(child) + + try: + while ready or active: + while ready and len(active) < jobs: + key = ready.popleft() + task = by_key[key] + failed = [dep for dep in task.dependencies if results[dep].returncode] + if failed: + complete(key, TaskResult(1, "prerequisite failed: " + ", ".join(failed))) + continue + log = None + try: + command = task.command() + task.log_path.parent.mkdir(parents=True, exist_ok=True) + log = task.log_path.open("wb") + metadata = {"command": command, "threads_per_job": threads, + "dependencies": task.dependencies} + records[key] = metadata + task.log_path.with_suffix(".execution.json").write_text(json.dumps(metadata, indent=2) + "\n") + process = subprocess.Popen(command, cwd=cwd, env=environment, + stdout=log, stderr=subprocess.STDOUT, start_new_session=True) + except (OSError, ValueError, RuntimeError) as exc: + if log is not None: + log.close() + complete(key, TaskResult(1, str(exc))) + continue + active[key] = process, log + print(f"[Started] {key}; log: {task.log_path}", flush=True) + completed = [(key, process.returncode) for key, (process, _) in active.items() + if process.poll() is not None] + for key, returncode in completed: + _, log = active.pop(key) + log.close() + complete(key, TaskResult(returncode)) + if active and not completed: + time.sleep(.05) + return results + finally: + stop_processes([process for process, _ in active.values()]) + for _, log in active.values(): + log.close() diff --git a/validation/tests/test_experiment_scheduler.py b/validation/tests/test_experiment_scheduler.py new file mode 100644 index 0000000..d629536 --- /dev/null +++ b/validation/tests/test_experiment_scheduler.py @@ -0,0 +1,187 @@ +from __future__ import annotations + +import argparse +import json +from pathlib import Path +import sys + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parents[1])) +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tools/pim/pimcomp/compare")) +from raptor_validation import experiment_scheduler as scheduler +import run_pimcomp_models as suite +import compare_raptor_pimcomp_model as comparator +sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tools/pim/ablation")) +import run_ablation as ablation + + +def task(root, key, code, dependencies=()): + command = [sys.executable, "-c", code] + return scheduler.ExperimentTask(key, lambda: command, dependencies, root / f"{key}.log") + + +def test_dependency_starts_before_unrelated_job_finishes(tmp_path): + # The slow job cannot finish until the fast job's dependent has run. + marker = tmp_path / "released" + slow = ("import pathlib,time; deadline=time.monotonic()+5; " + f"p=pathlib.Path({str(marker)!r}); " + "exec('while not p.exists() and time.monotonic() 1: - output.parent.mkdir(parents=True, exist_ok=True) - output.write_text('{"new": true}') - return [], offset + len(jobs) + def run_jobs(tasks, jobs, threads, cwd): + assert jobs == 1 + results = {} + for task in tasks: + try: + command = task.command() + except ValueError as exc: + errors.append(str(exc)) + results[task.key] = suite.TaskResult(1, str(exc)) + continue + if "--prepare-common" not in command: + commands.append(command) + output = Path(command[command.index("--out-dir") + 1]) / "pimcomp/comparison_report.json" + if fresh or only == "raptor": + output.parent.mkdir(parents=True, exist_ok=True) + output.write_text('{"new": true}') + results[task.key] = suite.TaskResult(0) + return results summary = tmp_path / "results.csv" summary.write_text("summary\n") monkeypatch.setattr(suite, "validate_pimcomp_source", lambda: None) - monkeypatch.setattr(suite, "run", lambda *a, **k: 0) - monkeypatch.setattr(suite, "run_comparison_jobs", run_jobs) + monkeypatch.setattr(suite, "run_tasks", run_jobs) monkeypatch.setattr(suite, "write_results_csv", lambda *a: summary) monkeypatch.setattr(suite, "comparison_passed", lambda report: True) monkeypatch.setattr(sys, "argv", [ @@ -548,15 +557,19 @@ def test_suite_reruns_anchor_and_shares_only_new_reports(tmp_path, monkeypatch, "--archs", "arch-a", "--mode", "throughput", "--jobs", "1", *(["--only", only] if only else []), ]) - assert suite.main() == 0 + assert suite.main() == (1 if only != "raptor" and not fresh else 0) if only == "raptor": - assert len(batches[0]) == 3 - assert all("--reuse-pimcomp-report" in command for command in batches[0]) + assert len(commands) == 3 + assert all("--reuse-pimcomp-report" in command for command in commands) else: - assert len(batches[0]) == 1 - assert "--reuse-pimcomp-report" not in batches[0][0] - assert all(("--reuse-pimcomp-report" in command) == fresh for command in batches[1]) - assert ("--reuse-raptor-report" in batches[0][0]) == (only == "pimcomp") + assert "--reuse-pimcomp-report" not in commands[0] + assert ("--reuse-raptor-report" in commands[0]) == (only == "pimcomp") + if fresh: + assert len(commands) == 3 + assert all("--reuse-pimcomp-report" in command for command in commands[1:]) + else: + assert len(commands) == 1 # Missing reports must not cause duplicate Pimcomp work. + assert len(errors) == 2 def test_missing_pimcomp_report_is_not_reused(tmp_path): @@ -658,7 +671,7 @@ def test_ablation_cli_selects_canonical_or_custom_output(tmp_path, monkeypatch, canonical = tmp_path / "scripts/results_ablation.csv" monkeypatch.setattr(ablation, "DEFAULT_OUT_DIR", artifact_root) monkeypatch.setattr(ablation, "DEFAULT_RESULTS_PATH", canonical) - monkeypatch.setattr(ablation.subprocess, "Popen", lambda *a, **k: argparse.Namespace(wait=lambda: 0)) + monkeypatch.setattr(suite, "run_plans", lambda plans: 0) calls = [] monkeypatch.setattr(ablation, "aggregate_results", lambda root, *a, **k: (calls.append((root, k["output"])) or None, [])) command = ["--variant", "baseline", "--jobs", "1"] @@ -863,7 +876,13 @@ def test_ablation_runner_distinguishes_invalid_outputs_from_failed_execution(tmp csv = tmp_path / "results.csv" csv.write_text("summary\n") monkeypatch.setattr(suite, "run", lambda *a, **k: 0) - monkeypatch.setattr(suite, "run_comparison_jobs", lambda *a: (["arch-a/vgg8/throughput/pipeline4"] if process_failed else [], 1)) + def complete_tasks(tasks, *args): + previous = output.stat().st_mtime_ns + output.write_text(json.dumps(report)) + os.utime(output, ns=(previous + 1000000, previous + 1000000)) + return {task.key: suite.TaskResult(int(process_failed)) for task in tasks} + + monkeypatch.setattr(suite, "run_tasks", complete_tasks) monkeypatch.setattr(suite, "write_results_csv", lambda *a: csv) monkeypatch.setattr(sys, "argv", ["run_pimcomp_models.py", "--out-dir", str(tmp_path), "--models", "vgg8", "--archs", "arch-a", "--mode", "throughput", "--pipeline", "4", diff --git a/validation/tools/pim/ablation/README.md b/validation/tools/pim/ablation/README.md index 5407482..1a27c0c 100644 --- a/validation/tools/pim/ablation/README.md +++ b/validation/tools/pim/ablation/README.md @@ -9,6 +9,9 @@ Run the synchronization/Spatial-planning ablation from the repository root: Defaults: `vgg8 resnet18 resnet34 googlenet`, `arch-a arch-b`, latency (pipeline 1), and throughput (pipeline 4). Select `yolo11n` or `arch-c` explicitly when needed. Only Raptor runs; Pimcomp validation is `SKIPPED`. +Ready models, architectures, modes, and variants run concurrently in one dependency queue. +References are prepared once per model. `--jobs` and `--threads-per-job` both default +to the available logical processor count; override them to limit CPU contention. | Variant | Raptor options | |---|---| @@ -34,7 +37,7 @@ suite pipelines for that mode. Pipeline-only selection uses its matching mode. `--pipeline auto` requests automatic throughput depth selection; positive numeric depths retain their exact-stage meaning. `--only` is rejected because -the wrapper always runs Raptor only. Ctrl+C stops the active variant's workers. +the wrapper always runs Raptor only. Ctrl+C stops all active experiment workers. Artifacts use `//artifacts//[/pipelineN]/ablation//`; reference inputs, outputs, and runners are shared under `artifacts/common/`. diff --git a/validation/tools/pim/ablation/run_ablation.py b/validation/tools/pim/ablation/run_ablation.py index 15cff0b..d58d138 100755 --- a/validation/tools/pim/ablation/run_ablation.py +++ b/validation/tools/pim/ablation/run_ablation.py @@ -6,10 +6,7 @@ from __future__ import annotations import argparse import csv import math -import os import shlex -import signal -import subprocess import sys from dataclasses import dataclass from pathlib import Path @@ -26,6 +23,7 @@ from raptor_validation.pimcomp_models import ( ) from raptor_validation.artifacts import remove_lock_files from raptor_validation.raptor import parse_pipeline_depth, pipeline_sort_key +from raptor_validation.experiment_scheduler import available_processors, positive_integer @dataclass(frozen=True) @@ -79,9 +77,14 @@ def parse_args(argv: list[str] | None = None) -> tuple[argparse.Namespace, list[ help="Run only this variant; baseline is added when needed. Repeat as needed.", ) parser.add_argument("--dry-run", action="store_true", help="Print runner commands without modifying files.") + parser.add_argument("--jobs", "-j", type=positive_integer, default=available_processors(), + help="Maximum concurrent cases (default: available logical processors).") + parser.add_argument("--threads-per-job", type=positive_integer, default=available_processors(), + help="Threads per case (default: available logical processors).") args, forwarded = parser.parse_known_args(argv) if any(option == "--only" or option.startswith("--only=") for option in forwarded): parser.error("--only is not supported; the ablation wrapper always runs Raptor only") + forwarded.extend(("--jobs", str(args.jobs), "--threads-per-job", str(args.threads_per_job))) return args, forwarded @@ -156,23 +159,6 @@ def runner_command( return command -def terminate_process_group(process: subprocess.Popen[bytes]) -> None: - if process.poll() is not None: - return - try: - os.killpg(process.pid, signal.SIGTERM) - except ProcessLookupError: - return - try: - process.wait(timeout=5) - except subprocess.TimeoutExpired: - try: - os.killpg(process.pid, signal.SIGKILL) - except ProcessLookupError: - return - process.wait() - - def percentage_difference( value: str | None, reference: str | None, *, lower_is_better: bool = False ) -> str: @@ -313,33 +299,27 @@ def main(argv: list[str] | None = None) -> int: ) return 0 - out_dir.mkdir(parents=True, exist_ok=True) - failed = [] - current_process = None + sys.path.insert(0, str(RUNNER.parent)) + import run_pimcomp_models as suite + plans = [] try: for variant in variants: - for case_label, case_forwarded in case_sets: + for _, case_forwarded in case_sets: if variant.name == "no-sync" and case_key(case_forwarded)[0] != "throughput": continue - command = runner_command( - out_dir, variant, args.models, case_forwarded, out_dir, False - ) - print(f"[{variant.name}/{case_label}] {shlex.join(command)}") - current_process = subprocess.Popen( - command, - cwd=REPO, - start_new_session=True, - ) - returncode = current_process.wait() - current_process = None - if returncode: - failed.append(f"{variant.name}/{case_label}: runner exited with {returncode}") + command = runner_command(out_dir, variant, args.models, case_forwarded, out_dir, False) + selected = suite.parse_args(command[2:]) + if selected.clean: + print(f"Removed {suite.clean_artifacts(out_dir, args.models, out_dir)} artifact path(s).") + return 0 + plans.append(suite.make_plan(selected)) + failed = ["experiment jobs failed"] if suite.run_plans(plans) else [] except KeyboardInterrupt: - if current_process is not None: - terminate_process_group(current_process) - remove_lock_files(out_dir) - print("Interrupted; terminated the active ablation job.", file=sys.stderr) + print("Interrupted; terminated ablation workers.", file=sys.stderr) return 130 + except ValueError as exc: + print(str(exc), file=sys.stderr) + return 2 remove_lock_files(out_dir) selected_cases = frozenset(case_key(arguments) for _, arguments in case_sets) diff --git a/validation/tools/pim/pimcomp/compare/README.md b/validation/tools/pim/pimcomp/compare/README.md index 34a1e8b..8c6aef8 100644 --- a/validation/tools/pim/pimcomp/compare/README.md +++ b/validation/tools/pim/pimcomp/compare/README.md @@ -11,7 +11,12 @@ built release Raptor, Pimcomp, and `pimsim-nn`. ``` Defaults: all five models, Arch-A/B, latency pipeline 1, throughput pipelines -2/4/8, four workers, and a functional throughput batch of 64. +2/4/8 and a functional throughput batch of 64. +Both `--jobs` and `--threads-per-job` default to the available logical processor count. +Their limits multiply: 40 jobs with 40 threads can create 1,600 threads. +Use `--jobs 8 --threads-per-job 4` to reduce contention. +The thread limit controls OpenMP, Raptor's compilation thread pool (`-j`), BLAS, and Rayon. +Reference batches use that many workers, with one thread per reference process. Use `--help` for the complete option reference. Numeric depths greater than one require exactly that many active compute stages. `--pipeline auto` requests throughput depth selection. Reports include `raptor_selected_pipeline`; the @@ -26,6 +31,8 @@ requested depth remains in the existing pipeline field. | `--only {raptor,pimcomp}` | Rerun one compiler and reuse the other's existing report. | | `--raptor-only` | Run Raptor without invoking Pimcomp. | | `--batch-size N`, `--jobs N` | Set functional throughput batch size and worker count. | +| `--threads-per-job N` | Set the thread limit inside each job. | +| `--with-ablation`, `--ablation-out-dir PATH` | Schedule both suites together; optionally select a separate ablation root. | | `--pimsim-time-ms N` | Throughput convergence deadline; default: 1000 ms. | | `--no-fast` | Use full-duration throughput simulation for authoritative experiments. | | `--timeout-seconds N` | Per-stage timeout; default: 0 (unlimited). | @@ -45,11 +52,19 @@ Artifacts use under `artifacts/common/` and `comparison_report.{md,json}` under each comparison's `pimcomp/`. YOLO and GoogLeNet use `artifacts/common-full-model/` to avoid reusing historical Softmax-free references. +References are prepared once per model. Each case starts when its own prerequisites finish. Pimcomp runs once per model/architecture/mode; later pipelines reuse its newly written report, including failures without simulation artifacts. Missing or -unchanged anchor reports trigger fresh attempts. The runner continues after failures and exits nonzero if a +unchanged prerequisite reports skip dependent cases without repeating Pimcomp work. +Skipped cases retain unavailable metrics rather than old measurements. +The runner continues after failures and exits nonzero if a compiler, simulation, or functional comparison fails. +Combined runs share references with ablation and reuse matching Raptor baseline reports. +`ablation/baseline/source_reports.json` records those report locations. +Only the parent process writes aggregate CSVs. Per-case logs and execution settings remain beside each case. +Thread limits are recorded in reports and checked during report reuse. + If fast-mode convergence fails, the reports and CSV retain valid full-duration fallback throughput and output interval values. This is not a converged steady-state result. Per-case JSON records `throughput_measurement: full_duration_fallback` and diff --git a/validation/tools/pim/pimcomp/compare/compare_raptor_pimcomp_model.py b/validation/tools/pim/pimcomp/compare/compare_raptor_pimcomp_model.py index c3086a3..b54327c 100755 --- a/validation/tools/pim/pimcomp/compare/compare_raptor_pimcomp_model.py +++ b/validation/tools/pim/pimcomp/compare/compare_raptor_pimcomp_model.py @@ -46,6 +46,7 @@ from raptor_validation.onnx_utils import ( ) from raptor_validation.raptor import compile_with_raptor, read_selected_pipeline_depth from raptor_validation.output_comparison import VALIDATION_POLICY, compare_output_arrays +from raptor_validation.experiment_scheduler import available_processors, positive_integer, thread_environment from raptor_validation.pimsim_nn import ( export_raptor_pimsim_artifact, parse_pimsim_nn_metrics, @@ -156,6 +157,7 @@ def run_logged( steps: list[StepRecord], stage: str | None = None, print_header: bool = True, + env: dict[str, str] | None = None, ) -> str: if print_header: print_step(name, cmd, cwd, stage) @@ -172,6 +174,7 @@ def run_logged( stderr=subprocess.STDOUT, text=True, timeout=None if timeout_sec == 0 else timeout_sec, + env=env, ) except subprocess.TimeoutExpired as exc: duration = time.perf_counter() - start @@ -364,7 +367,8 @@ def compile_reference( run_logged( "Compile reference ONNX IR", [str(args.raptor_path), str(model_path), "-o", str(onnx_ir_base), "--EmitONNXIR", - "--mlir-elide-elementsattrs-if-larger=16", "--enable-conv-opt-pass=false"], + "--mlir-elide-elementsattrs-if-larger=16", "--enable-conv-opt-pass=false", + f"-j={args.threads_per_job}"], cwd=REPO, timeout_sec=args.timeout_seconds, steps=steps, @@ -372,7 +376,7 @@ def compile_reference( ) run_logged( "Compile reference native", - [str(args.raptor_path), "-O3", str(model_path), "-o", str(runner_base)], + [str(args.raptor_path), "-O3", str(model_path), "-o", str(runner_base), f"-j={args.threads_per_job}"], cwd=REPO, timeout_sec=args.timeout_seconds, steps=steps, @@ -400,7 +404,7 @@ def compile_reference( ) run_logged( "Build runner", - ["cmake", "--build", ".", "-j"], + ["cmake", "--build", ".", "--parallel", str(args.threads_per_job)], cwd=build_dir, timeout_sec=args.timeout_seconds, steps=steps, @@ -446,6 +450,7 @@ def generate_reference_outputs( timeout_sec=args.timeout_seconds, steps=steps, print_header=print_header, + env=thread_environment(1), ) return reference_dir @@ -576,6 +581,7 @@ def compile_raptor_target( f"--core-count={hardware['core_count']}", f"--pim-target-config={args.pimcomp_config}", "--pim-emit-json", + f"-j={args.threads_per_job}", *args.raptor_extra_arg, ] print_step("Compile Raptor", cmd, REPO, "Compile Pim") @@ -584,6 +590,7 @@ def compile_raptor_target( raptor_extra_args = [ f"--pim-target-config={args.pimcomp_config}", "--pim-emit-json", + f"-j={args.threads_per_job}", *args.raptor_extra_arg, ] try: @@ -1202,6 +1209,7 @@ def experiment_context(args, model, functional_model, hardware, side): "oracle": args.pimsim_nn_build_dir / "ChipTest", "comparison": Path(__file__), "validation": VALIDATION_DIR / "raptor_validation/output_comparison.py", + "thread_policy": VALIDATION_DIR / "raptor_validation/experiment_scheduler.py", "functional_simulator": args.pim_simulator_dir / "target/release/pim-simulator"} if side == "raptor": files["compiler"] = args.raptor_path @@ -1216,6 +1224,7 @@ def experiment_context(args, model, functional_model, hardware, side): settings.update({name: getattr(args, name) for name in ( ("raptor_extra_arg",) if side == "raptor" else ("pimcomp_pipeline", "pimcomp_replication", "pimcomp_model_name"))}) + settings["threads_per_job"] = getattr(args, "threads_per_job", available_processors()) return {"version": 2, "validation_policy": VALIDATION_POLICY, "files": fingerprint_files(files), "hardware": hardware, "settings": settings} @@ -1569,9 +1578,9 @@ def main(): parser.add_argument( "-j", "--jobs", - type=int, - default=4, - help="Maximum parallel native reference runner processes (default: 4).", + type=positive_integer, + default=available_processors(), + help="Maximum parallel native reference runner processes (default: logical processors).", ) parser.add_argument("--pimcomp-pipeline", choices=["element", "batch"]) parser.add_argument("--pimcomp-model-name", help="Use a Pimcomp built-in model name such as vgg16.") @@ -1613,7 +1622,10 @@ def main(): action="store_true", help="Return a non-zero status if a stage or semantic validation fails.", ) + parser.add_argument("--threads-per-job", type=positive_integer, default=available_processors(), + help="Threads for OpenMP, MLIR compilation, and simulation (default: logical processors).") args = parser.parse_args() + os.environ.update(thread_environment(args.threads_per_job)) if args.raptor_only and any( option is not None for option in ( @@ -2152,6 +2164,7 @@ def main(): "hardware": hardware, "pimsim_mode": args.pimsim_mode, "pimsim_time_ms": args.pimsim_time_ms, + "threads_per_job": args.threads_per_job, "pimcomp_pipeline": args.pimcomp_pipeline, "pimcomp_replication": args.pimcomp_replication, "pimcomp_model_source": "not_run" if args.raptor_only else "original_onnx", diff --git a/validation/tools/pim/pimcomp/compare/run_pimcomp_models.py b/validation/tools/pim/pimcomp/compare/run_pimcomp_models.py index 00d60b0..9af71a6 100755 --- a/validation/tools/pim/pimcomp/compare/run_pimcomp_models.py +++ b/validation/tools/pim/pimcomp/compare/run_pimcomp_models.py @@ -5,16 +5,12 @@ import argparse import csv import json import math -import os import shlex import shutil import subprocess import sys -from concurrent.futures import ProcessPoolExecutor, as_completed -from contextlib import nullcontext, redirect_stderr, redirect_stdout from dataclasses import dataclass from pathlib import Path -from tempfile import TemporaryDirectory from colorama import Fore, Style @@ -31,7 +27,9 @@ from raptor_validation.artifacts import artifacts_dir, remove_lock_files from raptor_validation.pimsim_nn import parse_pimsim_nn_metrics from raptor_validation.output_comparison import VALIDATION_POLICY from raptor_validation.raptor import parse_pipeline_depth, pipeline_sort_key -from raptor_validation.validate_one import STAGE_COLORS +from raptor_validation.experiment_scheduler import ( + ExperimentTask, TaskResult, available_processors, positive_integer, run_tasks, +) PIMCOMP_SOURCE = REPO / "third_party/PIMCOMP-NN" PIMCOMP_CONFIGS = REPO / "validation/pimsim_configs/pimcomp" @@ -130,6 +128,8 @@ def write_results_csv( models: list[str], comparisons: tuple[tuple[str, int, str], ...] = COMPARISONS, ablation_variant: str | None = None, + report_paths: dict[Path, Path | None] | None = None, + raptor_only: bool = False, ) -> Path: output = DEFAULT_RESULTS_PATH if root is None else root / RESULTS_FILENAME if ablation_variant is not None: @@ -179,6 +179,9 @@ def write_results_csv( report_path = result_dir( root, name, arch, comparison_mode, pipeline, ablation_variant ) / "pimcomp/comparison_report.json" + if report_paths is not None: + directory = result_dir(root, name, arch, comparison_mode, pipeline, ablation_variant) + report_path = report_paths.get(directory, report_path) row = { "model": name, "arch": arch, @@ -198,12 +201,12 @@ def write_results_csv( "better_compiler": "NA", "speedup": "NA", } - if report_path.exists(): + if report_path is not None and report_path.exists(): report = json.loads(report_path.read_text(encoding="utf-8")) if report.get("validation_policy") != VALIDATION_POLICY: raise ValueError(f"Report uses a different validation policy: {report_path}. Regenerate this comparison.") raptor_values = performance_values(report.get("raptor_performance") or {}) - pimcomp_values = performance_values(report.get("pimcomp_performance") or {}) + pimcomp_values = performance_values({} if raptor_only else report.get("pimcomp_performance") or {}) raptor_metric = raptor_values["throughput"] if comparison_mode == "throughput" else raptor_values["latency"] pimcomp_metric = pimcomp_values["throughput"] if comparison_mode == "throughput" else pimcomp_values["latency"] row.update( @@ -211,7 +214,7 @@ def write_results_csv( report.get("raptor_validation") ), pimcomp_functional_validation=functional_validation_status( - report.get("pimcomp_validation") + {"status": "skipped"} if raptor_only else report.get("pimcomp_validation") ), raptor_throughput_samples_s=format_value(raptor_values["throughput"]), pimcomp_throughput_samples_s=format_value(pimcomp_values["throughput"]), @@ -322,17 +325,6 @@ def format_value(value: float | None) -> str: return f"{value:.6g}" -def print_stage(title: str, color: str) -> None: - print(Style.BRIGHT + color + f"[{title}]" + Style.RESET_ALL, flush=True) - - -def print_completed(label: str, output: str = "") -> None: - print(Style.BRIGHT + Fore.CYAN + f"[Completed {label}]" + Style.RESET_ALL) - if output: - print(output, end="" if output.endswith("\n") else "\n") - print("=" * 72, flush=True) - - def run(command: list[str], *, dry_run: bool, check: bool = True) -> int: print(f" cwd: {REPO}", flush=True) print(f" $ {shlex.join(command)}", flush=True) @@ -341,29 +333,6 @@ def run(command: list[str], *, dry_run: bool, check: bool = True) -> int: return subprocess.run(command, cwd=REPO, check=check).returncode -def run_comparison_job(job: tuple[str, list[str], Path | None]) -> tuple[str, int, str | None]: - label, command, log_path = job - sys.stdout.flush() - sys.stderr.flush() - if log_path is None: - return label, run(command, dry_run=False, check=False), None - - saved_stdout = os.dup(1) - saved_stderr = os.dup(2) - try: - with open(log_path, "w", encoding="utf-8", buffering=1) as log: - os.dup2(log.fileno(), 1) - os.dup2(log.fileno(), 2) - with redirect_stdout(log), redirect_stderr(log): - returncode = run(command, dry_run=False, check=False) - finally: - os.dup2(saved_stdout, 1) - os.dup2(saved_stderr, 2) - os.close(saved_stdout) - os.close(saved_stderr) - return label, returncode, log_path.read_text(encoding="utf-8", errors="replace") - - def validate_pimcomp_source() -> None: header = PIMCOMP_SOURCE / "backend/GeneticAlgorithm.h" source = header.read_text(encoding="utf-8") @@ -469,7 +438,7 @@ def comparison_command_for( reuse_pimcomp_report = report elif reuse_shared_pimcomp: reuse_pimcomp_report = spec.shared_pimcomp_dir.parent / "comparison_report.json" - return comparison_command( + command = comparison_command( spec.model, spec.functional_model, spec.output_dir, @@ -491,6 +460,7 @@ def comparison_command_for( ), reuse_pimcomp_report=reuse_pimcomp_report, ) + return command + ["--jobs", "1", "--threads-per-job", str(args.threads_per_job)] def pimcomp_report_stamp(output_dir: Path) -> tuple[int, int, int] | None: @@ -506,40 +476,6 @@ def pimcomp_report_ready(output_dir: Path, previous: tuple[int, int, int] | None return stamp is not None and stamp != previous -def run_comparison_jobs( - comparison_jobs: list[tuple[str, list[str], Path | None]], - max_workers: int, - log_dir: Path | None, - log_offset: int, -) -> tuple[list[str], int]: - if not comparison_jobs: - return [], log_offset - print_directly = min(max_workers, len(comparison_jobs)) == 1 - jobs = [ - ( - label, - command, - None if print_directly or log_dir is None else log_dir / f"{log_offset + index}.log", - ) - for index, (label, command, _) in enumerate(comparison_jobs) - ] - with (nullcontext(None) if print_directly else ProcessPoolExecutor(max_workers=max_workers)) as executor: - completed = ( - map(run_comparison_job, jobs) - if print_directly - else ( - future.result() - for future in as_completed(executor.submit(run_comparison_job, job) for job in jobs) - ) - ) - failed = [] - for label, returncode, output in completed: - print_completed(label, output or "") - if returncode: - failed.append(label) - return failed, log_offset + len(jobs) - - def ensure_throughput_config(arch: str, sim_time_ms: int) -> Path: source = PIMCOMP_CONFIGS / arch / "throughput_config_1000ms.json" target = PIMCOMP_CONFIGS / arch / f"throughput_config_{sim_time_ms}ms.json" @@ -566,7 +502,7 @@ def config_path(arch: str, mode: str, sim_time_ms: int, *, write: bool) -> Path: return path -def main() -> int: +def parse_args(argv: list[str] | None = None) -> argparse.Namespace: parser = argparse.ArgumentParser( description="Compare supported Pimcomp models with Raptor latency and throughput schedules." ) @@ -635,9 +571,9 @@ def main() -> int: parser.add_argument( "-j", "--jobs", - type=int, - default=4, - help="Number of comparisons to run in parallel (default: 4).", + type=positive_integer, + default=available_processors(), + help="Maximum concurrent comparisons (default: available logical processors).", ) parser.add_argument( "--clean", @@ -657,7 +593,11 @@ def main() -> int: dest="raptor_extra_args", help="Additional argument to pass to Raptor; repeat as needed.", ) - args = parser.parse_args() + parser.add_argument("--threads-per-job", type=positive_integer, default=available_processors(), + help="Threads per compiler/simulator job (default: available logical processors).") + parser.add_argument("--with-ablation", action="store_true", help="Run comparison and ablation in one dependency queue.") + parser.add_argument("--ablation-out-dir", type=Path, help="Output root for a combined ablation run.") + args = parser.parse_args(argv) if args.ablation_variant and Path(args.ablation_variant).name != args.ablation_variant: parser.error("--ablation-variant must be a single directory name") @@ -665,25 +605,35 @@ def main() -> int: if args.only is not None and args.raptor_only: parser.error("--only cannot be combined with --raptor-only") + return args + + +@dataclass +class SuitePlan: + args: argparse.Namespace + root: Path | None + common_root: Path | None + comparisons: dict + reference_batch_size: int + specs: list[ComparisonSpec] + aliases: dict[Path, Path] + previous_reports: dict[Path, tuple[int, int, int] | None] + + +def make_plan(args: argparse.Namespace) -> SuitePlan: out_dir = args.out_dir.resolve() if args.out_dir is not None else None common_root = args.common_dir.resolve() if args.common_dir is not None else None args.archs = list(dict.fromkeys(args.archs)) + args.models = list(dict.fromkeys(args.models)) args.mode = list(dict.fromkeys(args.mode)) - if args.clean: - print( - f"Removed {clean_artifacts(out_dir, args.models, common_root)} " - "comparison artifact path(s)." - ) - return 0 - if args.jobs < 1: - parser.error("--jobs must be at least 1") + raise ValueError("--jobs must be at least 1") if args.pimsim_time_ms <= 0: - parser.error("--pimsim-time-ms must be positive") + raise ValueError("--pimsim-time-ms must be positive") if args.batch_size <= 0: - parser.error("--batch-size must be positive") + raise ValueError("--batch-size must be positive") if args.timeout_seconds < 0: - parser.error("--timeout-seconds must be non-negative") + raise ValueError("--timeout-seconds must be non-negative") comparisons_by_arch: dict[str, tuple[tuple[str, int | str, str], ...]] = {} configs_by_arch: dict[str, dict[str, Path]] = {} for arch in args.archs: @@ -697,7 +647,7 @@ def main() -> int: and (args.pipeline is None or comparison[1] == args.pipeline) ) if not comparisons: - parser.error("--pipeline does not belong to the selected --mode") + raise ValueError("--pipeline does not belong to the selected --mode") configs = { mode: config_path(arch, mode, args.pimsim_time_ms, write=not args.dry_run) for mode, _, _ in comparisons @@ -716,7 +666,7 @@ def main() -> int: if not MODELS[name].exists() ] if missing: - parser.error(f"missing model(s): {', '.join(missing)}") + raise ValueError(f"missing model(s): {', '.join(missing)}") if args.only is not None: missing_reuse = [] @@ -730,7 +680,7 @@ def main() -> int: if not required.exists(): missing_reuse.append(str(required)) if missing_reuse: - parser.error( + raise ValueError( f"--only {args.only} needs existing comparison artifacts: " + ", ".join(missing_reuse) ) @@ -740,41 +690,6 @@ def main() -> int: if out_dir is not None and not args.dry_run: out_dir.mkdir(parents=True, exist_ok=True) - print(Style.BRIGHT + f"Found {len(args.models)} Pimcomp model(s) to compare." + Style.RESET_ALL) - print(f"Architectures: {', '.join(args.archs)}") - print(f"Modes: {', '.join(args.mode)}") - print(f"Throughput Pimsim time: {args.pimsim_time_ms} ms") - print(f"Max parallel jobs: {args.jobs}") - print(f"Shared reference batch: {reference_batch_size}") - print( - f"Comparison jobs: " - f"{sum(len(args.models) * len(comparisons) for comparisons in comparisons_by_arch.values())}" - ) - print(f"Artifacts root: {out_dir or SUITE}") - print("=" * 72) - - print_stage("Prepare shared artifacts", STAGE_COLORS["Build runner"]) - try: - for name in args.models: - try: - run( - prepare_common_command( - MODELS[name], - common_dir(out_dir, name, common_root), - args.timeout_seconds, - reference_batch_size, - args.jobs, - ), - dry_run=args.dry_run, - ) - finally: - print_completed(f"shared artifacts: {name}") - except KeyboardInterrupt: - remove_lock_files(out_dir or SUITE) - print("Interrupted; cleaned validation lock files.", file=sys.stderr) - return 130 - - failed = [] comparison_specs: list[ComparisonSpec] = [] shared_pimcomp_by_group: dict[tuple[str, str, str], Path] = {} for arch in args.archs: @@ -808,117 +723,171 @@ def main() -> int: ) ) - if args.dry_run: - for spec in comparison_specs: - command = comparison_command_for( - spec, - args, - reuse_shared_pimcomp=not spec.anchor, - ) - if run(command, dry_run=True, check=False): + return SuitePlan(args, out_dir, common_root, comparisons_by_arch, + reference_batch_size, comparison_specs, {}, {}) + + +def execution_key(plan: SuitePlan, spec: ComparisonSpec) -> tuple: + args = plan.args + return (spec.model, spec.common_dir, spec.config, spec.mode, spec.pipeline, + tuple(args.raptor_extra_args), args.batch_size, args.no_fast, + args.threads_per_job) + + +def build_tasks(plans: list[SuitePlan]) -> list[ExperimentTask]: + tasks = [] + references = {} + comparison_sources = {execution_key(plan, spec): spec.output_dir + for plan in plans if not plan.args.ablation_variant and plan.args.only is None + and not plan.args.raptor_only for spec in plan.specs} + for plan in plans: + for name in plan.args.models: + common = common_dir(plan.root, name, plan.common_root) + previous = references.get(common) + if previous is None or previous[0] < plan.reference_batch_size: + references[common] = (plan.reference_batch_size, name, plan.args) + for common, (batch_size, name, args) in references.items(): + command = prepare_common_command(MODELS[name], common, args.timeout_seconds, + batch_size, args.threads_per_job) + command += ["--threads-per-job", str(args.threads_per_job)] + tasks.append(ExperimentTask("reference:" + str(common), lambda command=command: command, + (), common / "prepare.log")) + for plan in plans: + args = plan.args + for spec in plan.specs: + source = comparison_sources.get(execution_key(plan, spec)) if args.ablation_variant == "baseline" else None + report_root = source if source is not None else spec.output_dir + plan.previous_reports[spec.output_dir] = pimcomp_report_stamp(report_root / "pimcomp/output") + if args.ablation_variant == "baseline" and execution_key(plan, spec) in comparison_sources: + plan.aliases[spec.output_dir] = comparison_sources[execution_key(plan, spec)] + continue + reuse = not (spec.anchor or args.only == "raptor" or args.raptor_only) + dependencies = ["reference:" + str(spec.common_dir)] + previous_report = pimcomp_report_stamp(spec.shared_pimcomp_dir) if reuse else None + if reuse: + dependencies.append(str(spec.shared_pimcomp_dir.parent.parent)) + + def command(spec=spec, args=args, reuse=reuse, previous=previous_report): + if reuse and not args.dry_run and not pimcomp_report_ready(spec.shared_pimcomp_dir, previous): + raise ValueError(f"Pimcomp prerequisite produced no fresh report: {spec.shared_pimcomp_dir}") + return comparison_command_for(spec, args, reuse_shared_pimcomp=reuse) + + tasks.append(ExperimentTask(str(spec.output_dir), command, tuple(dependencies), + spec.output_dir / "comparison.log")) + return tasks + + +def report_for(plan: SuitePlan, spec: ComparisonSpec) -> Path: + return plan.aliases.get(spec.output_dir, spec.output_dir) / "pimcomp/comparison_report.json" + + +def finish_plan(plan: SuitePlan, results: dict[str, TaskResult]) -> int: + args = plan.args + failed, diagnostic = [], [] + overrides = {} + for spec in plan.specs: + directory = plan.aliases.get(spec.output_dir, spec.output_dir) + result = results.get(str(directory)) + fresh = pimcomp_report_ready(directory / "pimcomp/output", plan.previous_reports[spec.output_dir]) + overrides[spec.output_dir] = report_for(plan, spec) if result and not result.returncode and fresh else None + for arch in args.archs: + output = write_results_csv(plan.root, arch, args.models, plan.comparisons[arch], + args.ablation_variant, overrides, args.raptor_only) + if plan.aliases: + destination = (plan.root or SUITE) / "ablation" / args.ablation_variant / "source_reports.json" + destination.write_text(json.dumps({str(k): str(v) if v is not None else None + for k, v in overrides.items()}, indent=2) + "\n") + for spec in plan.specs: + result = results.get(str(plan.aliases.get(spec.output_dir, spec.output_dir))) + path = report_for(plan, spec) + report = json.loads(path.read_text()) if overrides[spec.output_dir] is not None else {} + if result is None or result.returncode or overrides[spec.output_dir] is None: + failed.append(spec.label) + elif not comparison_passed(report): + if args.ablation_variant == "no-sync" and completed_synchronization_ablation(report): + diagnostic.append(spec.label) + else: failed.append(spec.label) - elif comparison_specs: - raptor_only_run = args.only == "raptor" or args.raptor_only - anchor_specs = comparison_specs if raptor_only_run else [ - spec for spec in comparison_specs if spec.anchor - ] - anchor_jobs = [ - ( - spec.label, - comparison_command_for(spec, args, reuse_shared_pimcomp=False), - None, - ) - for spec in anchor_specs - ] - dependent_specs = [] if raptor_only_run else [ - spec for spec in comparison_specs if not spec.anchor - ] - print_directly = min(args.jobs, len(comparison_specs)) == 1 - try: - with (nullcontext(None) if print_directly else TemporaryDirectory(prefix="raptor-pimcomp-")) as log_dir: - previous_reports = { - spec.shared_pimcomp_dir: pimcomp_report_stamp(spec.shared_pimcomp_dir) - for spec in anchor_specs - } - anchor_failed, log_offset = run_comparison_jobs( - anchor_jobs, - args.jobs, - Path(log_dir) if log_dir else None, - 0, - ) - failed.extend(anchor_failed) - dependent_jobs = [ - ( - spec.label, - comparison_command_for( - spec, - args, - reuse_shared_pimcomp=pimcomp_report_ready( - spec.shared_pimcomp_dir, previous_reports[spec.shared_pimcomp_dir] - ), - ), - None, - ) - for spec in dependent_specs - ] - dependent_failed, _ = run_comparison_jobs( - dependent_jobs, - args.jobs, - Path(log_dir) if log_dir else None, - log_offset, - ) - failed.extend(dependent_failed) - except KeyboardInterrupt: - remove_lock_files(out_dir or SUITE) - print("Interrupted; cleaned validation lock files.", file=sys.stderr) - return 130 - - if args.dry_run: - return 1 if failed else 0 - - remove_lock_files(out_dir or SUITE) - - results_path = None - for arch in args.archs: - results_path = write_results_csv( - out_dir, - arch, - args.models, - comparisons_by_arch[arch], - args.ablation_variant, - ) - assert results_path is not None - print_stage(results_path.name, STAGE_COLORS["Compare outputs"]) - print(results_path.read_text(encoding="utf-8"), end="") - diagnostic_failures = [] - for arch in args.archs: - for name in args.models: - for mode, pipeline, _ in comparisons_by_arch[arch]: - label = f"{arch}/{name}/{mode}/pipeline{pipeline}" - report_path = result_dir( - out_dir, name, arch, mode, pipeline, args.ablation_variant - ) / "pimcomp/comparison_report.json" - report = json.loads(report_path.read_text(encoding="utf-8")) if report_path.exists() else {} - if not comparison_passed(report): - if (label not in failed and args.ablation_variant == "no-sync" - and completed_synchronization_ablation(report)): - diagnostic_failures.append(label) - elif label not in failed: - failed.append(label) - print(Style.BRIGHT + Fore.CYAN + "Summary" + Style.RESET_ALL) - total_jobs = sum(len(args.models) * len(comparisons) for comparisons in comparisons_by_arch.values()) - print(Style.BRIGHT + f"Passed: {total_jobs - len(failed) - len(diagnostic_failures)}" + Style.RESET_ALL) - print(Style.BRIGHT + f"Failed: {len(failed)}" + Style.RESET_ALL) - if diagnostic_failures: - print("No-sync numerical failures (diagnostic only): " + ", ".join(diagnostic_failures)) - print(Style.BRIGHT + f"Results: {results_path}" + Style.RESET_ALL) + print(f"Results: {output}; passed: {len(plan.specs) - len(failed) - len(diagnostic)}; " + f"failed: {len(failed)}; diagnostic no-sync failures: {len(diagnostic)}", flush=True) if failed: - print( - Fore.RED + f"Failed comparisons: {', '.join(failed)}" + Style.RESET_ALL, - file=sys.stderr, - ) - return 1 - return 0 + print("Failed comparisons: " + ", ".join(failed), file=sys.stderr) + remove_lock_files(plan.root or SUITE) + return int(bool(failed)) + + +def run_plans(plans: list[SuitePlan]) -> int: + tasks = build_tasks(plans) + args = plans[0].args + if any((plan.args.jobs, plan.args.threads_per_job) != (args.jobs, args.threads_per_job) for plan in plans): + raise ValueError("all experiment plans must use the same job and thread limits") + if args.dry_run: + for task in tasks: + run(task.command(), dry_run=True, check=False) + return 0 + print(f"Concurrent jobs: {args.jobs}; threads per job: {args.threads_per_job}", flush=True) + results = run_tasks(tasks, args.jobs, args.threads_per_job, REPO) + # Only the parent writes aggregate CSVs. + failures = [finish_plan(plan, results) for plan in plans] + return int(any(failures)) + + +def combined_ablation_plans(args: argparse.Namespace) -> tuple[list[SuitePlan], object, Path]: + sys.path.insert(0, str(REPO / "validation/tools/pim/ablation")) + import run_ablation as ablation + from copy import deepcopy + root = args.ablation_out_dir or args.out_dir or SUITE + models = [name for name in args.models if name in ablation.ABLATION_DEFAULT_MODELS] + if not models: + raise ValueError("combined ablation needs at least one of the four ablation models") + cases = [(mode, depth) for mode, depth, _ in COMPARISONS + if mode in args.mode and (depth in (1, 4) if args.pipeline is None else depth == args.pipeline)] + if args.pipeline is not None and not cases: + cases = [("throughput", args.pipeline)] + plans = [] + for variant in ablation.VARIANTS: + for mode, depth in cases: + if variant.name == "no-sync" and mode != "throughput": + continue + selected = deepcopy(args) + selected.out_dir, selected.models = root, models + selected.common_dir = args.common_dir or args.out_dir or SUITE + selected.raptor_only, selected.only = True, None + selected.ablation_variant = variant.name + selected.mode, selected.pipeline = [mode], depth + selected.raptor_extra_args += list(variant.raptor_args) + plans.append(make_plan(selected)) + return plans, ablation, root + + +def main(argv: list[str] | None = None) -> int: + args = parse_args(argv) + if args.clean: + print(f"Removed {clean_artifacts(args.out_dir, args.models, args.common_dir)} comparison artifact path(s).") + return 0 + try: + plans = [make_plan(args)] + if args.with_ablation: + if args.only or args.raptor_only or args.ablation_variant: + raise ValueError("--with-ablation requires a full compiler comparison") + extra, ablation, ablation_root = combined_ablation_plans(args) + plans.extend(extra) + result = run_plans(plans) + if args.with_ablation and not args.dry_run: + cases = frozenset((spec.mode, str(spec.pipeline)) for plan in extra for spec in plan.specs) + variants = [variant for variant in ablation.VARIANTS + if any(plan.args.ablation_variant == variant.name for plan in extra)] + _, failures = ablation.aggregate_results(ablation_root, variants, cases, + frozenset(extra[0].args.models), + output=ablation.DEFAULT_RESULTS_PATH if args.out_dir is None and args.ablation_out_dir is None else None) + result = int(bool(result or failures)) + return result + except KeyboardInterrupt: + print("Interrupted; terminated experiment workers.", file=sys.stderr) + return 130 + except ValueError as exc: + print(str(exc), file=sys.stderr) + return 2 if __name__ == "__main__":