Files
Niccolò Nicolosi 66db4b66da
Validate Operations / validate-operations (push) Has been cancelled
add paper artifact
2026-11-03 14:37:25 +01:00

141 lines
6.9 KiB
Python

"""Container commands for the pinned Raptor paper artifact."""
from __future__ import annotations
import argparse
import csv
from datetime import datetime, timezone
from html import escape
import json
import os
from pathlib import Path
import platform
import shutil
import subprocess
import sys
import time
HERE = Path(__file__).resolve().parent
REPO = Path('/opt/raptor')
PYTHON = REPO / '.venv/bin/python'
COMPARE = Path('validation/tools/pim/pimcomp/compare')
ABLATION = Path('validation/tools/pim/ablation/run_ablation.py')
def stage(name: str, command: list[str], root: Path) -> dict:
print(f'[{name}] {command}', flush=True)
start = time.monotonic()
with (root / f'{name}.log').open('wb') as log:
with subprocess.Popen(command, cwd=REPO, stdout=subprocess.PIPE,
stderr=subprocess.STDOUT) as process:
for line in process.stdout:
sys.stdout.buffer.write(line)
sys.stdout.buffer.flush()
log.write(line)
returncode = process.wait()
return {'command': command, 'exit_code': returncode,
'seconds': round(time.monotonic() - start, 3)}
def plots(root: Path, smoke: bool) -> dict:
commands = {'ablation-plots': [str(PYTHON), str(HERE / 'plot_ablation.py'),
'--input', str(root / 'ablation/results_ablation.csv'),
'--output-dir', str(root / 'plots/ablation')]}
if not smoke:
commands['comparison-plots'] = [str(PYTHON), str(COMPARE / 'generate_plots.py'),
'--input', str(root / 'comparison/results_comparison.csv'),
'--output-dir', str(root / 'plots/comparison')]
return {name: stage(name, command, root) for name, command in commands.items()}
def show_results(root: Path) -> None:
page = ['<!doctype html><html lang="en"><meta charset="utf-8">',
'<meta name="viewport" content="width=device-width, initial-scale=1">',
'<title>Raptor results</title><style>',
'body{font:16px sans-serif;max-width:1100px;margin:2rem auto;padding:0 1rem}',
'img{max-width:100%}table{border-collapse:collapse;font-size:13px}',
'th,td{padding:6px;border:1px solid #ccc;white-space:nowrap}',
'.table{overflow-x:auto}</style><h1>Raptor results</h1>',
'<p>Functional validation status is shown in the tables and plots.</p>']
for title, relative in (
('Comparison', 'plots/comparison/png/performance_energy_compact.png'),
('Pipeline throughput', 'plots/comparison/png/raptor_pipeline_throughput.png'),
('Ablation', 'plots/ablation/ablation.png'),
):
if (root / relative).is_file():
page.append(f'<h2>{title}</h2><a href="{relative}"><img src="{relative}" alt="{title}"></a>')
for title, relative in (('Comparison', 'comparison/results_comparison.csv'),
('Ablation', 'ablation/results_ablation.csv')):
path = root / relative
if path.is_file():
page.append(f'<h2>{title} data</h2><p><a href="{relative}">Download CSV</a></p><div class="table"><table>')
with path.open(newline='') as stream:
for index, row in enumerate(csv.reader(stream)):
tag = 'th' if index == 0 else 'td'
page.append('<tr>' + ''.join(f'<{tag}>{escape(value)}</{tag}>' for value in row) + '</tr>')
page.append('</table></div>')
page.append('<h2>Files</h2><ul>')
for path in sorted([*root.glob('*.log'), *root.glob('plots/*')]):
relative = path.relative_to(root).as_posix()
page.append(f'<li><a href="{relative}">{escape(relative)}</a></li>')
page.append('</ul></html>')
(root / 'index.html').write_text('\n'.join(page) + '\n')
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument('command', choices=('reproduce', 'smoke', 'plots'))
parser.add_argument('--smoke', action='store_true', help='Plot the smoke results.')
parser.add_argument('--jobs', type=int, default=int(os.environ.get('JOBS', len(os.sched_getaffinity(0)))))
parser.add_argument('--threads-per-job', type=int,
default=int(os.environ.get('THREADS_PER_JOB', len(os.sched_getaffinity(0)))))
args = parser.parse_args()
smoke = args.command == 'smoke' or args.smoke
root = Path('/results') / ('smoke' if smoke else 'full')
os.environ['CARGO_NET_OFFLINE'] = 'true'
if args.command == 'plots':
manifest_path = root / 'run.json'
manifest = json.loads(manifest_path.read_text())
records = plots(root, smoke)
manifest['stages'].update(records)
manifest_path.write_text(json.dumps(manifest, indent=2) + '\n')
show_results(root)
return int(any(record['exit_code'] for record in records.values()))
if root.exists():
parser.error(f'{root} exists. Select a fresh OUTPUT_DIR to prevent stale result reuse.')
jobs = args.jobs
if jobs < 1 or args.threads_per_job < 1:
parser.error('jobs and threads-per-job must be positive')
root.mkdir(parents=True)
shutil.copytree(HERE / 'provenance', root / 'provenance')
# Copy the artifact scripts so each result identifies its orchestration code.
shutil.copytree(HERE, root / 'provenance/artifact',
ignore=shutil.ignore_patterns('provenance', '__pycache__'))
manifest = {'started_utc': datetime.now(timezone.utc).isoformat(),
'profile': 'smoke' if smoke else 'full', 'image_id': os.environ.get('ARTIFACT_IMAGE_ID'),
'platform': platform.platform(), 'machine': platform.machine(),
'cpu_count': os.cpu_count(), 'jobs': jobs, 'threads_per_job': args.threads_per_job,
'batch_size': 64, 'fast': False, 'stages': {}}
common = ['--jobs', str(jobs), '--threads-per-job', str(args.threads_per_job),
'--batch-size', '64', '--no-fast']
subset = ['--models', 'vgg8', '--archs', 'arch-a', '--mode', 'latency', '--pipeline', '1'] if smoke else []
commands = {
'experiments': [str(PYTHON), str(COMPARE / 'run_pimcomp_models.py'),
'--out-dir', str(root / 'comparison'),
'--with-ablation', '--ablation-out-dir', str(root / 'ablation'),
'--common-dir', str(root / 'references'), *common, *subset],
}
manifest_path = root / 'run.json'
manifest_path.write_text(json.dumps(manifest, indent=2) + '\n')
for name, command in commands.items():
manifest['stages'][name] = stage(name, command, root)
manifest_path.write_text(json.dumps(manifest, indent=2) + '\n')
manifest['stages'].update(plots(root, smoke))
manifest['finished_utc'] = datetime.now(timezone.utc).isoformat()
manifest_path.write_text(json.dumps(manifest, indent=2) + '\n')
show_results(root)
return int(any(record['exit_code'] for record in manifest['stages'].values()))
if __name__ == '__main__':
raise SystemExit(main())