141 lines
6.9 KiB
Python
141 lines
6.9 KiB
Python
"""Container commands for the pinned Raptor paper artifact."""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import csv
|
|
from datetime import datetime, timezone
|
|
from html import escape
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import platform
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
HERE = Path(__file__).resolve().parent
|
|
REPO = Path('/opt/raptor')
|
|
PYTHON = REPO / '.venv/bin/python'
|
|
COMPARE = Path('validation/tools/pim/pimcomp/compare')
|
|
ABLATION = Path('validation/tools/pim/ablation/run_ablation.py')
|
|
|
|
|
|
def stage(name: str, command: list[str], root: Path) -> dict:
|
|
print(f'[{name}] {command}', flush=True)
|
|
start = time.monotonic()
|
|
with (root / f'{name}.log').open('wb') as log:
|
|
with subprocess.Popen(command, cwd=REPO, stdout=subprocess.PIPE,
|
|
stderr=subprocess.STDOUT) as process:
|
|
for line in process.stdout:
|
|
sys.stdout.buffer.write(line)
|
|
sys.stdout.buffer.flush()
|
|
log.write(line)
|
|
returncode = process.wait()
|
|
return {'command': command, 'exit_code': returncode,
|
|
'seconds': round(time.monotonic() - start, 3)}
|
|
|
|
|
|
def plots(root: Path, smoke: bool) -> dict:
|
|
commands = {'ablation-plots': [str(PYTHON), str(HERE / 'plot_ablation.py'),
|
|
'--input', str(root / 'ablation/results_ablation.csv'),
|
|
'--output-dir', str(root / 'plots/ablation')]}
|
|
if not smoke:
|
|
commands['comparison-plots'] = [str(PYTHON), str(COMPARE / 'generate_plots.py'),
|
|
'--input', str(root / 'comparison/results_comparison.csv'),
|
|
'--output-dir', str(root / 'plots/comparison')]
|
|
return {name: stage(name, command, root) for name, command in commands.items()}
|
|
|
|
|
|
def show_results(root: Path) -> None:
|
|
page = ['<!doctype html><html lang="en"><meta charset="utf-8">',
|
|
'<meta name="viewport" content="width=device-width, initial-scale=1">',
|
|
'<title>Raptor results</title><style>',
|
|
'body{font:16px sans-serif;max-width:1100px;margin:2rem auto;padding:0 1rem}',
|
|
'img{max-width:100%}table{border-collapse:collapse;font-size:13px}',
|
|
'th,td{padding:6px;border:1px solid #ccc;white-space:nowrap}',
|
|
'.table{overflow-x:auto}</style><h1>Raptor results</h1>',
|
|
'<p>Functional validation status is shown in the tables and plots.</p>']
|
|
for title, relative in (
|
|
('Comparison', 'plots/comparison/png/performance_energy_compact.png'),
|
|
('Pipeline throughput', 'plots/comparison/png/raptor_pipeline_throughput.png'),
|
|
('Ablation', 'plots/ablation/ablation.png'),
|
|
):
|
|
if (root / relative).is_file():
|
|
page.append(f'<h2>{title}</h2><a href="{relative}"><img src="{relative}" alt="{title}"></a>')
|
|
for title, relative in (('Comparison', 'comparison/results_comparison.csv'),
|
|
('Ablation', 'ablation/results_ablation.csv')):
|
|
path = root / relative
|
|
if path.is_file():
|
|
page.append(f'<h2>{title} data</h2><p><a href="{relative}">Download CSV</a></p><div class="table"><table>')
|
|
with path.open(newline='') as stream:
|
|
for index, row in enumerate(csv.reader(stream)):
|
|
tag = 'th' if index == 0 else 'td'
|
|
page.append('<tr>' + ''.join(f'<{tag}>{escape(value)}</{tag}>' for value in row) + '</tr>')
|
|
page.append('</table></div>')
|
|
page.append('<h2>Files</h2><ul>')
|
|
for path in sorted([*root.glob('*.log'), *root.glob('plots/*')]):
|
|
relative = path.relative_to(root).as_posix()
|
|
page.append(f'<li><a href="{relative}">{escape(relative)}</a></li>')
|
|
page.append('</ul></html>')
|
|
(root / 'index.html').write_text('\n'.join(page) + '\n')
|
|
|
|
|
|
def main() -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__)
|
|
parser.add_argument('command', choices=('reproduce', 'smoke', 'plots'))
|
|
parser.add_argument('--smoke', action='store_true', help='Plot the smoke results.')
|
|
parser.add_argument('--jobs', type=int, default=int(os.environ.get('JOBS', len(os.sched_getaffinity(0)))))
|
|
parser.add_argument('--threads-per-job', type=int,
|
|
default=int(os.environ.get('THREADS_PER_JOB', len(os.sched_getaffinity(0)))))
|
|
args = parser.parse_args()
|
|
smoke = args.command == 'smoke' or args.smoke
|
|
root = Path('/results') / ('smoke' if smoke else 'full')
|
|
os.environ['CARGO_NET_OFFLINE'] = 'true'
|
|
if args.command == 'plots':
|
|
manifest_path = root / 'run.json'
|
|
manifest = json.loads(manifest_path.read_text())
|
|
records = plots(root, smoke)
|
|
manifest['stages'].update(records)
|
|
manifest_path.write_text(json.dumps(manifest, indent=2) + '\n')
|
|
show_results(root)
|
|
return int(any(record['exit_code'] for record in records.values()))
|
|
if root.exists():
|
|
parser.error(f'{root} exists. Select a fresh OUTPUT_DIR to prevent stale result reuse.')
|
|
jobs = args.jobs
|
|
if jobs < 1 or args.threads_per_job < 1:
|
|
parser.error('jobs and threads-per-job must be positive')
|
|
root.mkdir(parents=True)
|
|
shutil.copytree(HERE / 'provenance', root / 'provenance')
|
|
# Copy the artifact scripts so each result identifies its orchestration code.
|
|
shutil.copytree(HERE, root / 'provenance/artifact',
|
|
ignore=shutil.ignore_patterns('provenance', '__pycache__'))
|
|
manifest = {'started_utc': datetime.now(timezone.utc).isoformat(),
|
|
'profile': 'smoke' if smoke else 'full', 'image_id': os.environ.get('ARTIFACT_IMAGE_ID'),
|
|
'platform': platform.platform(), 'machine': platform.machine(),
|
|
'cpu_count': os.cpu_count(), 'jobs': jobs, 'threads_per_job': args.threads_per_job,
|
|
'batch_size': 64, 'fast': False, 'stages': {}}
|
|
common = ['--jobs', str(jobs), '--threads-per-job', str(args.threads_per_job),
|
|
'--batch-size', '64', '--no-fast']
|
|
subset = ['--models', 'vgg8', '--archs', 'arch-a', '--mode', 'latency', '--pipeline', '1'] if smoke else []
|
|
commands = {
|
|
'experiments': [str(PYTHON), str(COMPARE / 'run_pimcomp_models.py'),
|
|
'--out-dir', str(root / 'comparison'),
|
|
'--with-ablation', '--ablation-out-dir', str(root / 'ablation'),
|
|
'--common-dir', str(root / 'references'), *common, *subset],
|
|
}
|
|
manifest_path = root / 'run.json'
|
|
manifest_path.write_text(json.dumps(manifest, indent=2) + '\n')
|
|
for name, command in commands.items():
|
|
manifest['stages'][name] = stage(name, command, root)
|
|
manifest_path.write_text(json.dumps(manifest, indent=2) + '\n')
|
|
manifest['stages'].update(plots(root, smoke))
|
|
manifest['finished_utc'] = datetime.now(timezone.utc).isoformat()
|
|
manifest_path.write_text(json.dumps(manifest, indent=2) + '\n')
|
|
show_results(root)
|
|
return int(any(record['exit_code'] for record in manifest['stages'].values()))
|
|
|
|
|
|
if __name__ == '__main__':
|
|
raise SystemExit(main())
|