Merge remote-tracking branch 'origin/main'
Validate Operations / validate-operations (push) Has been cancelled

This commit is contained in:
NiccoloN
2026-11-05 15:54:00 +01:00
21 changed files with 1161 additions and 311 deletions
@@ -0,0 +1,153 @@
"""Run experiment subprocesses as their dependencies complete."""
from __future__ import annotations
import argparse
from collections import deque
from dataclasses import dataclass
import json
import os
from pathlib import Path
import signal
import subprocess
import time
from typing import Callable
def available_processors() -> int:
if hasattr(os, "sched_getaffinity"):
return len(os.sched_getaffinity(0))
return os.cpu_count() or 1
def positive_integer(value: str) -> int:
number = int(value)
if number < 1:
raise argparse.ArgumentTypeError("must be positive")
return number
def thread_environment(threads: int) -> dict[str, str]:
return {**os.environ, "OMP_NUM_THREADS": str(threads),
"OMP_THREAD_LIMIT": str(threads), "OMP_DYNAMIC": "FALSE",
"OMP_WAIT_POLICY": "PASSIVE", "OPENBLAS_NUM_THREADS": str(threads),
"MKL_NUM_THREADS": str(threads), "RAYON_NUM_THREADS": str(threads),
"CMAKE_BUILD_PARALLEL_LEVEL": str(threads), "CARGO_BUILD_JOBS": str(threads)}
@dataclass(frozen=True)
class ExperimentTask:
key: str
command: Callable[[], list[str]]
dependencies: tuple[str, ...]
log_path: Path
@dataclass(frozen=True)
class TaskResult:
returncode: int
error: str = ""
def stop_processes(processes: list[subprocess.Popen]) -> None:
for process in processes:
try:
os.killpg(process.pid, signal.SIGTERM)
except ProcessLookupError:
pass
deadline = time.monotonic() + 5
for process in processes:
try:
process.wait(timeout=max(0, deadline - time.monotonic()))
except subprocess.TimeoutExpired:
try:
os.killpg(process.pid, signal.SIGKILL)
except ProcessLookupError:
pass
process.wait()
def run_tasks(tasks: list[ExperimentTask], jobs: int, threads: int,
cwd: Path) -> dict[str, TaskResult]:
if jobs < 1 or threads < 1:
raise ValueError("jobs and threads-per-job must be positive")
by_key = {task.key: task for task in tasks}
if len(by_key) != len(tasks):
raise ValueError("duplicate experiment task")
waiting = {task.key: set(task.dependencies) for task in tasks}
children = {key: [] for key in by_key}
for task in tasks:
for dependency in task.dependencies:
if dependency not in by_key:
raise ValueError(f"unknown dependency: {dependency}")
children[dependency].append(task.key)
ready = deque(key for key in by_key if not waiting[key])
degrees = {key: len(dependencies) for key, dependencies in waiting.items()}
pending = deque(ready)
visited = 0
while pending:
visited += 1
for child in children[pending.popleft()]:
degrees[child] -= 1
if not degrees[child]:
pending.append(child)
if visited != len(tasks):
raise ValueError("experiment dependency cycle")
active = {}
results = {}
records = {}
environment = thread_environment(threads)
def complete(key: str, result: TaskResult) -> None:
results[key] = result
task = by_key[key]
record = records.get(key, {"dependencies": task.dependencies, "threads_per_job": threads})
record.update(returncode=result.returncode, error=result.error)
task.log_path.parent.mkdir(parents=True, exist_ok=True)
task.log_path.with_suffix(".execution.json").write_text(json.dumps(record, indent=2) + "\n")
print(f"[{'Failed' if result.returncode else 'Completed'}] {key}"
f"{': ' + result.error if result.error else ''}", flush=True)
for child in children[key]:
waiting[child].remove(key)
if not waiting[child]:
ready.append(child)
try:
while ready or active:
while ready and len(active) < jobs:
key = ready.popleft()
task = by_key[key]
failed = [dep for dep in task.dependencies if results[dep].returncode]
if failed:
complete(key, TaskResult(1, "prerequisite failed: " + ", ".join(failed)))
continue
log = None
try:
command = task.command()
task.log_path.parent.mkdir(parents=True, exist_ok=True)
log = task.log_path.open("wb")
metadata = {"command": command, "threads_per_job": threads,
"dependencies": task.dependencies}
records[key] = metadata
task.log_path.with_suffix(".execution.json").write_text(json.dumps(metadata, indent=2) + "\n")
process = subprocess.Popen(command, cwd=cwd, env=environment,
stdout=log, stderr=subprocess.STDOUT, start_new_session=True)
except (OSError, ValueError, RuntimeError) as exc:
if log is not None:
log.close()
complete(key, TaskResult(1, str(exc)))
continue
active[key] = process, log
print(f"[Started] {key}; log: {task.log_path}", flush=True)
completed = [(key, process.returncode) for key, (process, _) in active.items()
if process.poll() is not None]
for key, returncode in completed:
_, log = active.pop(key)
log.close()
complete(key, TaskResult(returncode))
if active and not completed:
time.sleep(.05)
return results
finally:
stop_processes([process for process, _ in active.values()])
for _, log in active.values():
log.close()
@@ -0,0 +1,187 @@
from __future__ import annotations
import argparse
import json
from pathlib import Path
import sys
import pytest
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tools/pim/pimcomp/compare"))
from raptor_validation import experiment_scheduler as scheduler
import run_pimcomp_models as suite
import compare_raptor_pimcomp_model as comparator
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "tools/pim/ablation"))
import run_ablation as ablation
def task(root, key, code, dependencies=()):
command = [sys.executable, "-c", code]
return scheduler.ExperimentTask(key, lambda: command, dependencies, root / f"{key}.log")
def test_dependency_starts_before_unrelated_job_finishes(tmp_path):
# The slow job cannot finish until the fast job's dependent has run.
marker = tmp_path / "released"
slow = ("import pathlib,time; deadline=time.monotonic()+5; "
f"p=pathlib.Path({str(marker)!r}); "
"exec('while not p.exists() and time.monotonic()<deadline: time.sleep(.01)'); "
"assert p.exists(), 'dependent was blocked by an unrelated job'")
tasks = [task(tmp_path, "fast", "pass"), task(tmp_path, "slow", slow),
task(tmp_path, "dependent", f"from pathlib import Path; Path({str(marker)!r}).touch()", ("fast",))]
results = scheduler.run_tasks(tasks, 2, 1, tmp_path)
assert all(result.returncode == 0 for result in results.values())
def test_failure_skips_dependents_but_runs_other_jobs(tmp_path):
marker = tmp_path / "must-not-run"
tasks = [task(tmp_path, "failed", "raise SystemExit(3)"),
task(tmp_path, "dependent", f"from pathlib import Path; Path({str(marker)!r}).touch()", ("failed",)),
task(tmp_path, "independent", "pass")]
results = scheduler.run_tasks(tasks, 2, 1, tmp_path)
assert results["failed"].returncode == 3
assert results["dependent"].returncode != 0
assert results["independent"].returncode == 0
assert not marker.exists()
def test_thread_environment_and_execution_record(tmp_path):
names = ["OMP_NUM_THREADS", "OMP_THREAD_LIMIT", "RAYON_NUM_THREADS",
"CMAKE_BUILD_PARALLEL_LEVEL", "CARGO_BUILD_JOBS"]
job = task(tmp_path, "environment", f"import os,json; print(json.dumps({{k:os.environ[k] for k in {names!r}}}))")
scheduler.run_tasks([job], 1, 3, tmp_path)
assert set(json.loads(job.log_path.read_text()).values()) == {"3"}
record = json.loads(job.log_path.with_suffix(".execution.json").read_text())
assert record["threads_per_job"] == 3
assert scheduler.thread_environment(3)["OMP_WAIT_POLICY"] == "PASSIVE"
def test_concurrent_process_limit(tmp_path):
jobs = []
for index in range(6):
output = tmp_path / f"interval-{index}.json"
code = ("import json,time; from pathlib import Path; start=time.monotonic_ns(); "
f"time.sleep(.05); Path({str(output)!r}).write_text(json.dumps([start,time.monotonic_ns()]))")
jobs.append(task(tmp_path, str(index), code))
results = scheduler.run_tasks(jobs, 2, 1, tmp_path)
assert all(result.returncode == 0 for result in results.values())
events = []
for path in tmp_path.glob("interval-*.json"):
start, finish = json.loads(path.read_text())
events.extend(((start, 1), (finish, -1)))
active = 0
for _, delta in sorted(events):
active += delta
assert active <= 2
assert len(events) == 12 and active == 0
def test_interrupt_terminates_workers(tmp_path, monkeypatch):
processes = []
popen = scheduler.subprocess.Popen
def start(*args, **kwargs):
process = popen(*args, **kwargs)
processes.append(process)
return process
monkeypatch.setattr(scheduler.subprocess, "Popen", start)
sleep = scheduler.time.sleep
interrupted = False
def interrupt_once(seconds):
nonlocal interrupted
if not interrupted:
interrupted = True
raise KeyboardInterrupt()
sleep(seconds)
monkeypatch.setattr(scheduler.time, "sleep", interrupt_once)
with pytest.raises(KeyboardInterrupt):
scheduler.run_tasks([task(tmp_path, "waiting", "import time; time.sleep(30)")], 1, 1, tmp_path)
assert processes and all(process.poll() is not None for process in processes)
def test_invalid_dependency_graph_is_rejected(tmp_path):
with pytest.raises(ValueError, match="unknown dependency"):
scheduler.run_tasks([task(tmp_path, "a", "pass", ("missing",))], 1, 1, tmp_path)
with pytest.raises(ValueError, match="cycle"):
scheduler.run_tasks([task(tmp_path, "a", "pass", ("b",)),
task(tmp_path, "b", "pass", ("a",))], 1, 1, tmp_path)
def test_defaults_follow_processor_count(monkeypatch):
monkeypatch.setattr(suite, "available_processors", lambda: 7)
args = suite.parse_args([])
assert args.jobs == args.threads_per_job == 7
args = suite.parse_args(["--jobs", "2", "--threads-per-job", "3"])
assert (args.jobs, args.threads_per_job) == (2, 3)
with pytest.raises(SystemExit):
suite.parse_args(["--threads-per-job", "0"])
monkeypatch.setattr(ablation, "available_processors", lambda: 7)
args, forwarded = ablation.parse_args([])
assert args.jobs == args.threads_per_job == 7
assert forwarded[forwarded.index("--threads-per-job") + 1] == "7"
def test_combined_plan_reuses_references_and_raptor_baselines(tmp_path, monkeypatch):
monkeypatch.setattr(suite, "validate_pimcomp_source", lambda: None)
args = suite.parse_args(["--out-dir", str(tmp_path / "comparison"),
"--ablation-out-dir", str(tmp_path / "ablation"), "--common-dir", str(tmp_path / "references"),
"--jobs", "2", "--threads-per-job", "2", "--dry-run"])
comparison = suite.make_plan(args)
ablations, _, _ = suite.combined_ablation_plans(args)
tasks = suite.build_tasks([comparison, *ablations])
assert len([job for job in tasks if job.key.startswith("reference:")]) == 5
assert len(tasks) == 5 + 40 + 24 # 16 baseline cases reuse comparison reports.
assert sum(len(plan.aliases) for plan in ablations) == 16
assert len({job.key for job in tasks}) == len(tasks)
for job in tasks:
if job.key.startswith("reference:"):
command = job.command()
assert command[command.index("--jobs") + 1] == "2"
continue
command = job.command()
assert command[command.index("--threads-per-job") + 1] == "2"
if "--reuse-pimcomp-report" in command:
assert len(job.dependencies) == 2
else:
assert len(job.dependencies) == 1
def test_raptor_receives_thread_limit(tmp_path, monkeypatch):
received = []
monkeypatch.setattr(comparator, "compile_with_raptor",
lambda *args, **kwargs: received.extend(kwargs["raptor_extra_args"]) or {})
args = argparse.Namespace(threads_per_job=3, pimcomp_config=tmp_path / "config.json",
raptor_extra_arg=[], raptor_path=tmp_path / "compiler", verbose_raptor_compile=False, timeout_seconds=0)
comparator.compile_raptor_target(tmp_path / "model.onnx", tmp_path / "output",
{"core_count": 1, "crossbar_count": 1, "crossbar_size": 128}, args, [])
assert "-j=3" in received
def test_stale_report_is_not_published_after_missing_execution(tmp_path):
args = suite.parse_args(["--out-dir", str(tmp_path), "--models", "vgg8",
"--archs", "arch-a", "--mode", "latency", "--raptor-only"])
plan = suite.make_plan(args)
report = suite.report_for(plan, plan.specs[0])
report.parent.mkdir(parents=True)
report.write_text(json.dumps({"validation_policy": suite.VALIDATION_POLICY,
"raptor_validation": {"status": "done", "passed": True},
"raptor_performance": {"latency_ms": 1}}))
tasks = suite.build_tasks([plan])
assert suite.finish_plan(plan, {task.key: suite.TaskResult(0) for task in tasks}) == 1
import csv
with (tmp_path / "results_comparison.csv").open() as stream:
row = next(csv.DictReader(stream))
assert row["raptor_latency_ms"] == row["raptor_functional_validation"] == "NA"
def test_reference_workers_do_not_create_nested_thread_teams(tmp_path, monkeypatch):
environments = []
monkeypatch.setattr(comparator, "save_inputs_to_files", lambda *a: ([], None))
monkeypatch.setattr(comparator, "run_logged", lambda *a, **k: environments.append(k["env"]))
comparator.generate_reference_outputs(tmp_path / "runner", tmp_path, tmp_path / "model.onnx", [], [],
argparse.Namespace(timeout_seconds=0), tmp_path / "output")
assert environments[0]["OMP_NUM_THREADS"] == environments[0]["OPENBLAS_NUM_THREADS"] == "1"
+40 -21
View File
@@ -524,23 +524,32 @@ def test_suite_reruns_anchor_and_shares_only_new_reports(tmp_path, monkeypatch,
report.parent.mkdir(parents=True)
report.write_text("{}")
os.utime(report, ns=(1, 1))
batches = []
commands = []
errors = []
def run_jobs(jobs, max_workers, log_dir, offset):
assert max_workers == 1
batches.append([command for _, command, _ in jobs])
for _, command, _ in jobs:
output = Path(command[command.index("--out-dir") + 1]) / "pimcomp/comparison_report.json"
if fresh or len(batches) > 1:
output.parent.mkdir(parents=True, exist_ok=True)
output.write_text('{"new": true}')
return [], offset + len(jobs)
def run_jobs(tasks, jobs, threads, cwd):
assert jobs == 1
results = {}
for task in tasks:
try:
command = task.command()
except ValueError as exc:
errors.append(str(exc))
results[task.key] = suite.TaskResult(1, str(exc))
continue
if "--prepare-common" not in command:
commands.append(command)
output = Path(command[command.index("--out-dir") + 1]) / "pimcomp/comparison_report.json"
if fresh or only == "raptor":
output.parent.mkdir(parents=True, exist_ok=True)
output.write_text('{"new": true}')
results[task.key] = suite.TaskResult(0)
return results
summary = tmp_path / "results.csv"
summary.write_text("summary\n")
monkeypatch.setattr(suite, "validate_pimcomp_source", lambda: None)
monkeypatch.setattr(suite, "run", lambda *a, **k: 0)
monkeypatch.setattr(suite, "run_comparison_jobs", run_jobs)
monkeypatch.setattr(suite, "run_tasks", run_jobs)
monkeypatch.setattr(suite, "write_results_csv", lambda *a: summary)
monkeypatch.setattr(suite, "comparison_passed", lambda report: True)
monkeypatch.setattr(sys, "argv", [
@@ -548,15 +557,19 @@ def test_suite_reruns_anchor_and_shares_only_new_reports(tmp_path, monkeypatch,
"--archs", "arch-a", "--mode", "throughput", "--jobs", "1",
*(["--only", only] if only else []),
])
assert suite.main() == 0
assert suite.main() == (1 if only != "raptor" and not fresh else 0)
if only == "raptor":
assert len(batches[0]) == 3
assert all("--reuse-pimcomp-report" in command for command in batches[0])
assert len(commands) == 3
assert all("--reuse-pimcomp-report" in command for command in commands)
else:
assert len(batches[0]) == 1
assert "--reuse-pimcomp-report" not in batches[0][0]
assert all(("--reuse-pimcomp-report" in command) == fresh for command in batches[1])
assert ("--reuse-raptor-report" in batches[0][0]) == (only == "pimcomp")
assert "--reuse-pimcomp-report" not in commands[0]
assert ("--reuse-raptor-report" in commands[0]) == (only == "pimcomp")
if fresh:
assert len(commands) == 3
assert all("--reuse-pimcomp-report" in command for command in commands[1:])
else:
assert len(commands) == 1 # Missing reports must not cause duplicate Pimcomp work.
assert len(errors) == 2
def test_missing_pimcomp_report_is_not_reused(tmp_path):
@@ -658,7 +671,7 @@ def test_ablation_cli_selects_canonical_or_custom_output(tmp_path, monkeypatch,
canonical = tmp_path / "scripts/results_ablation.csv"
monkeypatch.setattr(ablation, "DEFAULT_OUT_DIR", artifact_root)
monkeypatch.setattr(ablation, "DEFAULT_RESULTS_PATH", canonical)
monkeypatch.setattr(ablation.subprocess, "Popen", lambda *a, **k: argparse.Namespace(wait=lambda: 0))
monkeypatch.setattr(suite, "run_plans", lambda plans: 0)
calls = []
monkeypatch.setattr(ablation, "aggregate_results", lambda root, *a, **k: (calls.append((root, k["output"])) or None, []))
command = ["--variant", "baseline", "--jobs", "1"]
@@ -863,7 +876,13 @@ def test_ablation_runner_distinguishes_invalid_outputs_from_failed_execution(tmp
csv = tmp_path / "results.csv"
csv.write_text("summary\n")
monkeypatch.setattr(suite, "run", lambda *a, **k: 0)
monkeypatch.setattr(suite, "run_comparison_jobs", lambda *a: (["arch-a/vgg8/throughput/pipeline4"] if process_failed else [], 1))
def complete_tasks(tasks, *args):
previous = output.stat().st_mtime_ns
output.write_text(json.dumps(report))
os.utime(output, ns=(previous + 1000000, previous + 1000000))
return {task.key: suite.TaskResult(int(process_failed)) for task in tasks}
monkeypatch.setattr(suite, "run_tasks", complete_tasks)
monkeypatch.setattr(suite, "write_results_csv", lambda *a: csv)
monkeypatch.setattr(sys, "argv", ["run_pimcomp_models.py", "--out-dir", str(tmp_path),
"--models", "vgg8", "--archs", "arch-a", "--mode", "throughput", "--pipeline", "4",
+4 -1
View File
@@ -9,6 +9,9 @@ Run the synchronization/Spatial-planning ablation from the repository root:
Defaults: `vgg8 resnet18 resnet34 googlenet`, `arch-a arch-b`, latency
(pipeline 1), and throughput (pipeline 4). Select `yolo11n` or `arch-c`
explicitly when needed. Only Raptor runs; Pimcomp validation is `SKIPPED`.
Ready models, architectures, modes, and variants run concurrently in one dependency queue.
References are prepared once per model. `--jobs` and `--threads-per-job` both default
to the available logical processor count; override them to limit CPU contention.
| Variant | Raptor options |
|---|---|
@@ -34,7 +37,7 @@ suite pipelines for that mode. Pipeline-only selection uses its matching mode.
`--pipeline auto` requests automatic throughput depth selection; positive numeric
depths retain their exact-stage meaning.
`--only` is rejected because
the wrapper always runs Raptor only. Ctrl+C stops the active variant's workers.
the wrapper always runs Raptor only. Ctrl+C stops all active experiment workers.
Artifacts use `<out-dir>/<model>/artifacts/<arch>/<mode>[/pipelineN]/ablation/<variant>/`;
reference inputs, outputs, and runners are shared under `artifacts/common/`.
+21 -41
View File
@@ -6,10 +6,7 @@ from __future__ import annotations
import argparse
import csv
import math
import os
import shlex
import signal
import subprocess
import sys
from dataclasses import dataclass
from pathlib import Path
@@ -26,6 +23,7 @@ from raptor_validation.pimcomp_models import (
)
from raptor_validation.artifacts import remove_lock_files
from raptor_validation.raptor import parse_pipeline_depth, pipeline_sort_key
from raptor_validation.experiment_scheduler import available_processors, positive_integer
@dataclass(frozen=True)
@@ -79,9 +77,14 @@ def parse_args(argv: list[str] | None = None) -> tuple[argparse.Namespace, list[
help="Run only this variant; baseline is added when needed. Repeat as needed.",
)
parser.add_argument("--dry-run", action="store_true", help="Print runner commands without modifying files.")
parser.add_argument("--jobs", "-j", type=positive_integer, default=available_processors(),
help="Maximum concurrent cases (default: available logical processors).")
parser.add_argument("--threads-per-job", type=positive_integer, default=available_processors(),
help="Threads per case (default: available logical processors).")
args, forwarded = parser.parse_known_args(argv)
if any(option == "--only" or option.startswith("--only=") for option in forwarded):
parser.error("--only is not supported; the ablation wrapper always runs Raptor only")
forwarded.extend(("--jobs", str(args.jobs), "--threads-per-job", str(args.threads_per_job)))
return args, forwarded
@@ -156,23 +159,6 @@ def runner_command(
return command
def terminate_process_group(process: subprocess.Popen[bytes]) -> None:
if process.poll() is not None:
return
try:
os.killpg(process.pid, signal.SIGTERM)
except ProcessLookupError:
return
try:
process.wait(timeout=5)
except subprocess.TimeoutExpired:
try:
os.killpg(process.pid, signal.SIGKILL)
except ProcessLookupError:
return
process.wait()
def percentage_difference(
value: str | None, reference: str | None, *, lower_is_better: bool = False
) -> str:
@@ -313,33 +299,27 @@ def main(argv: list[str] | None = None) -> int:
)
return 0
out_dir.mkdir(parents=True, exist_ok=True)
failed = []
current_process = None
sys.path.insert(0, str(RUNNER.parent))
import run_pimcomp_models as suite
plans = []
try:
for variant in variants:
for case_label, case_forwarded in case_sets:
for _, case_forwarded in case_sets:
if variant.name == "no-sync" and case_key(case_forwarded)[0] != "throughput":
continue
command = runner_command(
out_dir, variant, args.models, case_forwarded, out_dir, False
)
print(f"[{variant.name}/{case_label}] {shlex.join(command)}")
current_process = subprocess.Popen(
command,
cwd=REPO,
start_new_session=True,
)
returncode = current_process.wait()
current_process = None
if returncode:
failed.append(f"{variant.name}/{case_label}: runner exited with {returncode}")
command = runner_command(out_dir, variant, args.models, case_forwarded, out_dir, False)
selected = suite.parse_args(command[2:])
if selected.clean:
print(f"Removed {suite.clean_artifacts(out_dir, args.models, out_dir)} artifact path(s).")
return 0
plans.append(suite.make_plan(selected))
failed = ["experiment jobs failed"] if suite.run_plans(plans) else []
except KeyboardInterrupt:
if current_process is not None:
terminate_process_group(current_process)
remove_lock_files(out_dir)
print("Interrupted; terminated the active ablation job.", file=sys.stderr)
print("Interrupted; terminated ablation workers.", file=sys.stderr)
return 130
except ValueError as exc:
print(str(exc), file=sys.stderr)
return 2
remove_lock_files(out_dir)
selected_cases = frozenset(case_key(arguments) for _, arguments in case_sets)
+17 -2
View File
@@ -11,7 +11,12 @@ built release Raptor, Pimcomp, and `pimsim-nn`.
```
Defaults: all five models, Arch-A/B, latency pipeline 1, throughput pipelines
2/4/8, four workers, and a functional throughput batch of 64.
2/4/8 and a functional throughput batch of 64.
Both `--jobs` and `--threads-per-job` default to the available logical processor count.
Their limits multiply: 40 jobs with 40 threads can create 1,600 threads.
Use `--jobs 8 --threads-per-job 4` to reduce contention.
The thread limit controls OpenMP, Raptor's compilation thread pool (`-j`), BLAS, and Rayon.
Reference batches use that many workers, with one thread per reference process.
Use `--help` for the complete option reference. Numeric depths greater than one
require exactly that many active compute stages. `--pipeline auto` requests
throughput depth selection. Reports include `raptor_selected_pipeline`; the
@@ -26,6 +31,8 @@ requested depth remains in the existing pipeline field.
| `--only {raptor,pimcomp}` | Rerun one compiler and reuse the other's existing report. |
| `--raptor-only` | Run Raptor without invoking Pimcomp. |
| `--batch-size N`, `--jobs N` | Set functional throughput batch size and worker count. |
| `--threads-per-job N` | Set the thread limit inside each job. |
| `--with-ablation`, `--ablation-out-dir PATH` | Schedule both suites together; optionally select a separate ablation root. |
| `--pimsim-time-ms N` | Throughput convergence deadline; default: 1000 ms. |
| `--no-fast` | Use full-duration throughput simulation for authoritative experiments. |
| `--timeout-seconds N` | Per-stage timeout; default: 0 (unlimited). |
@@ -45,11 +52,19 @@ Artifacts use
under `artifacts/common/`
and `comparison_report.{md,json}` under each comparison's `pimcomp/`.
YOLO and GoogLeNet use `artifacts/common-full-model/` to avoid reusing historical Softmax-free references.
References are prepared once per model. Each case starts when its own prerequisites finish.
Pimcomp runs once per model/architecture/mode; later pipelines reuse its newly
written report, including failures without simulation artifacts. Missing or
unchanged anchor reports trigger fresh attempts. The runner continues after failures and exits nonzero if a
unchanged prerequisite reports skip dependent cases without repeating Pimcomp work.
Skipped cases retain unavailable metrics rather than old measurements.
The runner continues after failures and exits nonzero if a
compiler, simulation, or functional comparison fails.
Combined runs share references with ablation and reuse matching Raptor baseline reports.
`ablation/baseline/source_reports.json` records those report locations.
Only the parent process writes aggregate CSVs. Per-case logs and execution settings remain beside each case.
Thread limits are recorded in reports and checked during report reuse.
If fast-mode convergence fails, the reports and CSV retain valid full-duration
fallback throughput and output interval values. This is not a converged steady-state result.
Per-case JSON records `throughput_measurement: full_duration_fallback` and
@@ -46,6 +46,7 @@ from raptor_validation.onnx_utils import (
)
from raptor_validation.raptor import compile_with_raptor, read_selected_pipeline_depth
from raptor_validation.output_comparison import VALIDATION_POLICY, compare_output_arrays
from raptor_validation.experiment_scheduler import available_processors, positive_integer, thread_environment
from raptor_validation.pimsim_nn import (
export_raptor_pimsim_artifact,
parse_pimsim_nn_metrics,
@@ -156,6 +157,7 @@ def run_logged(
steps: list[StepRecord],
stage: str | None = None,
print_header: bool = True,
env: dict[str, str] | None = None,
) -> str:
if print_header:
print_step(name, cmd, cwd, stage)
@@ -172,6 +174,7 @@ def run_logged(
stderr=subprocess.STDOUT,
text=True,
timeout=None if timeout_sec == 0 else timeout_sec,
env=env,
)
except subprocess.TimeoutExpired as exc:
duration = time.perf_counter() - start
@@ -364,7 +367,8 @@ def compile_reference(
run_logged(
"Compile reference ONNX IR",
[str(args.raptor_path), str(model_path), "-o", str(onnx_ir_base), "--EmitONNXIR",
"--mlir-elide-elementsattrs-if-larger=16", "--enable-conv-opt-pass=false"],
"--mlir-elide-elementsattrs-if-larger=16", "--enable-conv-opt-pass=false",
f"-j={args.threads_per_job}"],
cwd=REPO,
timeout_sec=args.timeout_seconds,
steps=steps,
@@ -372,7 +376,7 @@ def compile_reference(
)
run_logged(
"Compile reference native",
[str(args.raptor_path), "-O3", str(model_path), "-o", str(runner_base)],
[str(args.raptor_path), "-O3", str(model_path), "-o", str(runner_base), f"-j={args.threads_per_job}"],
cwd=REPO,
timeout_sec=args.timeout_seconds,
steps=steps,
@@ -400,7 +404,7 @@ def compile_reference(
)
run_logged(
"Build runner",
["cmake", "--build", ".", "-j"],
["cmake", "--build", ".", "--parallel", str(args.threads_per_job)],
cwd=build_dir,
timeout_sec=args.timeout_seconds,
steps=steps,
@@ -446,6 +450,7 @@ def generate_reference_outputs(
timeout_sec=args.timeout_seconds,
steps=steps,
print_header=print_header,
env=thread_environment(1),
)
return reference_dir
@@ -576,6 +581,7 @@ def compile_raptor_target(
f"--core-count={hardware['core_count']}",
f"--pim-target-config={args.pimcomp_config}",
"--pim-emit-json",
f"-j={args.threads_per_job}",
*args.raptor_extra_arg,
]
print_step("Compile Raptor", cmd, REPO, "Compile Pim")
@@ -584,6 +590,7 @@ def compile_raptor_target(
raptor_extra_args = [
f"--pim-target-config={args.pimcomp_config}",
"--pim-emit-json",
f"-j={args.threads_per_job}",
*args.raptor_extra_arg,
]
try:
@@ -1202,6 +1209,7 @@ def experiment_context(args, model, functional_model, hardware, side):
"oracle": args.pimsim_nn_build_dir / "ChipTest",
"comparison": Path(__file__),
"validation": VALIDATION_DIR / "raptor_validation/output_comparison.py",
"thread_policy": VALIDATION_DIR / "raptor_validation/experiment_scheduler.py",
"functional_simulator": args.pim_simulator_dir / "target/release/pim-simulator"}
if side == "raptor":
files["compiler"] = args.raptor_path
@@ -1216,6 +1224,7 @@ def experiment_context(args, model, functional_model, hardware, side):
settings.update({name: getattr(args, name) for name in (
("raptor_extra_arg",) if side == "raptor" else
("pimcomp_pipeline", "pimcomp_replication", "pimcomp_model_name"))})
settings["threads_per_job"] = getattr(args, "threads_per_job", available_processors())
return {"version": 2, "validation_policy": VALIDATION_POLICY,
"files": fingerprint_files(files), "hardware": hardware, "settings": settings}
@@ -1569,9 +1578,9 @@ def main():
parser.add_argument(
"-j",
"--jobs",
type=int,
default=4,
help="Maximum parallel native reference runner processes (default: 4).",
type=positive_integer,
default=available_processors(),
help="Maximum parallel native reference runner processes (default: logical processors).",
)
parser.add_argument("--pimcomp-pipeline", choices=["element", "batch"])
parser.add_argument("--pimcomp-model-name", help="Use a Pimcomp built-in model name such as vgg16.")
@@ -1613,7 +1622,10 @@ def main():
action="store_true",
help="Return a non-zero status if a stage or semantic validation fails.",
)
parser.add_argument("--threads-per-job", type=positive_integer, default=available_processors(),
help="Threads for OpenMP, MLIR compilation, and simulation (default: logical processors).")
args = parser.parse_args()
os.environ.update(thread_environment(args.threads_per_job))
if args.raptor_only and any(
option is not None
for option in (
@@ -2152,6 +2164,7 @@ def main():
"hardware": hardware,
"pimsim_mode": args.pimsim_mode,
"pimsim_time_ms": args.pimsim_time_ms,
"threads_per_job": args.threads_per_job,
"pimcomp_pipeline": args.pimcomp_pipeline,
"pimcomp_replication": args.pimcomp_replication,
"pimcomp_model_source": "not_run" if args.raptor_only else "original_onnx",
@@ -5,16 +5,12 @@ import argparse
import csv
import json
import math
import os
import shlex
import shutil
import subprocess
import sys
from concurrent.futures import ProcessPoolExecutor, as_completed
from contextlib import nullcontext, redirect_stderr, redirect_stdout
from dataclasses import dataclass
from pathlib import Path
from tempfile import TemporaryDirectory
from colorama import Fore, Style
@@ -31,7 +27,9 @@ from raptor_validation.artifacts import artifacts_dir, remove_lock_files
from raptor_validation.pimsim_nn import parse_pimsim_nn_metrics
from raptor_validation.output_comparison import VALIDATION_POLICY
from raptor_validation.raptor import parse_pipeline_depth, pipeline_sort_key
from raptor_validation.validate_one import STAGE_COLORS
from raptor_validation.experiment_scheduler import (
ExperimentTask, TaskResult, available_processors, positive_integer, run_tasks,
)
PIMCOMP_SOURCE = REPO / "third_party/PIMCOMP-NN"
PIMCOMP_CONFIGS = REPO / "validation/pimsim_configs/pimcomp"
@@ -130,6 +128,8 @@ def write_results_csv(
models: list[str],
comparisons: tuple[tuple[str, int, str], ...] = COMPARISONS,
ablation_variant: str | None = None,
report_paths: dict[Path, Path | None] | None = None,
raptor_only: bool = False,
) -> Path:
output = DEFAULT_RESULTS_PATH if root is None else root / RESULTS_FILENAME
if ablation_variant is not None:
@@ -179,6 +179,9 @@ def write_results_csv(
report_path = result_dir(
root, name, arch, comparison_mode, pipeline, ablation_variant
) / "pimcomp/comparison_report.json"
if report_paths is not None:
directory = result_dir(root, name, arch, comparison_mode, pipeline, ablation_variant)
report_path = report_paths.get(directory, report_path)
row = {
"model": name,
"arch": arch,
@@ -198,12 +201,12 @@ def write_results_csv(
"better_compiler": "NA",
"speedup": "NA",
}
if report_path.exists():
if report_path is not None and report_path.exists():
report = json.loads(report_path.read_text(encoding="utf-8"))
if report.get("validation_policy") != VALIDATION_POLICY:
raise ValueError(f"Report uses a different validation policy: {report_path}. Regenerate this comparison.")
raptor_values = performance_values(report.get("raptor_performance") or {})
pimcomp_values = performance_values(report.get("pimcomp_performance") or {})
pimcomp_values = performance_values({} if raptor_only else report.get("pimcomp_performance") or {})
raptor_metric = raptor_values["throughput"] if comparison_mode == "throughput" else raptor_values["latency"]
pimcomp_metric = pimcomp_values["throughput"] if comparison_mode == "throughput" else pimcomp_values["latency"]
row.update(
@@ -211,7 +214,7 @@ def write_results_csv(
report.get("raptor_validation")
),
pimcomp_functional_validation=functional_validation_status(
report.get("pimcomp_validation")
{"status": "skipped"} if raptor_only else report.get("pimcomp_validation")
),
raptor_throughput_samples_s=format_value(raptor_values["throughput"]),
pimcomp_throughput_samples_s=format_value(pimcomp_values["throughput"]),
@@ -322,17 +325,6 @@ def format_value(value: float | None) -> str:
return f"{value:.6g}"
def print_stage(title: str, color: str) -> None:
print(Style.BRIGHT + color + f"[{title}]" + Style.RESET_ALL, flush=True)
def print_completed(label: str, output: str = "") -> None:
print(Style.BRIGHT + Fore.CYAN + f"[Completed {label}]" + Style.RESET_ALL)
if output:
print(output, end="" if output.endswith("\n") else "\n")
print("=" * 72, flush=True)
def run(command: list[str], *, dry_run: bool, check: bool = True) -> int:
print(f" cwd: {REPO}", flush=True)
print(f" $ {shlex.join(command)}", flush=True)
@@ -341,29 +333,6 @@ def run(command: list[str], *, dry_run: bool, check: bool = True) -> int:
return subprocess.run(command, cwd=REPO, check=check).returncode
def run_comparison_job(job: tuple[str, list[str], Path | None]) -> tuple[str, int, str | None]:
label, command, log_path = job
sys.stdout.flush()
sys.stderr.flush()
if log_path is None:
return label, run(command, dry_run=False, check=False), None
saved_stdout = os.dup(1)
saved_stderr = os.dup(2)
try:
with open(log_path, "w", encoding="utf-8", buffering=1) as log:
os.dup2(log.fileno(), 1)
os.dup2(log.fileno(), 2)
with redirect_stdout(log), redirect_stderr(log):
returncode = run(command, dry_run=False, check=False)
finally:
os.dup2(saved_stdout, 1)
os.dup2(saved_stderr, 2)
os.close(saved_stdout)
os.close(saved_stderr)
return label, returncode, log_path.read_text(encoding="utf-8", errors="replace")
def validate_pimcomp_source() -> None:
header = PIMCOMP_SOURCE / "backend/GeneticAlgorithm.h"
source = header.read_text(encoding="utf-8")
@@ -469,7 +438,7 @@ def comparison_command_for(
reuse_pimcomp_report = report
elif reuse_shared_pimcomp:
reuse_pimcomp_report = spec.shared_pimcomp_dir.parent / "comparison_report.json"
return comparison_command(
command = comparison_command(
spec.model,
spec.functional_model,
spec.output_dir,
@@ -491,6 +460,7 @@ def comparison_command_for(
),
reuse_pimcomp_report=reuse_pimcomp_report,
)
return command + ["--jobs", "1", "--threads-per-job", str(args.threads_per_job)]
def pimcomp_report_stamp(output_dir: Path) -> tuple[int, int, int] | None:
@@ -506,40 +476,6 @@ def pimcomp_report_ready(output_dir: Path, previous: tuple[int, int, int] | None
return stamp is not None and stamp != previous
def run_comparison_jobs(
comparison_jobs: list[tuple[str, list[str], Path | None]],
max_workers: int,
log_dir: Path | None,
log_offset: int,
) -> tuple[list[str], int]:
if not comparison_jobs:
return [], log_offset
print_directly = min(max_workers, len(comparison_jobs)) == 1
jobs = [
(
label,
command,
None if print_directly or log_dir is None else log_dir / f"{log_offset + index}.log",
)
for index, (label, command, _) in enumerate(comparison_jobs)
]
with (nullcontext(None) if print_directly else ProcessPoolExecutor(max_workers=max_workers)) as executor:
completed = (
map(run_comparison_job, jobs)
if print_directly
else (
future.result()
for future in as_completed(executor.submit(run_comparison_job, job) for job in jobs)
)
)
failed = []
for label, returncode, output in completed:
print_completed(label, output or "")
if returncode:
failed.append(label)
return failed, log_offset + len(jobs)
def ensure_throughput_config(arch: str, sim_time_ms: int) -> Path:
source = PIMCOMP_CONFIGS / arch / "throughput_config_1000ms.json"
target = PIMCOMP_CONFIGS / arch / f"throughput_config_{sim_time_ms}ms.json"
@@ -566,7 +502,7 @@ def config_path(arch: str, mode: str, sim_time_ms: int, *, write: bool) -> Path:
return path
def main() -> int:
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Compare supported Pimcomp models with Raptor latency and throughput schedules."
)
@@ -635,9 +571,9 @@ def main() -> int:
parser.add_argument(
"-j",
"--jobs",
type=int,
default=4,
help="Number of comparisons to run in parallel (default: 4).",
type=positive_integer,
default=available_processors(),
help="Maximum concurrent comparisons (default: available logical processors).",
)
parser.add_argument(
"--clean",
@@ -657,7 +593,11 @@ def main() -> int:
dest="raptor_extra_args",
help="Additional argument to pass to Raptor; repeat as needed.",
)
args = parser.parse_args()
parser.add_argument("--threads-per-job", type=positive_integer, default=available_processors(),
help="Threads per compiler/simulator job (default: available logical processors).")
parser.add_argument("--with-ablation", action="store_true", help="Run comparison and ablation in one dependency queue.")
parser.add_argument("--ablation-out-dir", type=Path, help="Output root for a combined ablation run.")
args = parser.parse_args(argv)
if args.ablation_variant and Path(args.ablation_variant).name != args.ablation_variant:
parser.error("--ablation-variant must be a single directory name")
@@ -665,25 +605,35 @@ def main() -> int:
if args.only is not None and args.raptor_only:
parser.error("--only cannot be combined with --raptor-only")
return args
@dataclass
class SuitePlan:
args: argparse.Namespace
root: Path | None
common_root: Path | None
comparisons: dict
reference_batch_size: int
specs: list[ComparisonSpec]
aliases: dict[Path, Path]
previous_reports: dict[Path, tuple[int, int, int] | None]
def make_plan(args: argparse.Namespace) -> SuitePlan:
out_dir = args.out_dir.resolve() if args.out_dir is not None else None
common_root = args.common_dir.resolve() if args.common_dir is not None else None
args.archs = list(dict.fromkeys(args.archs))
args.models = list(dict.fromkeys(args.models))
args.mode = list(dict.fromkeys(args.mode))
if args.clean:
print(
f"Removed {clean_artifacts(out_dir, args.models, common_root)} "
"comparison artifact path(s)."
)
return 0
if args.jobs < 1:
parser.error("--jobs must be at least 1")
raise ValueError("--jobs must be at least 1")
if args.pimsim_time_ms <= 0:
parser.error("--pimsim-time-ms must be positive")
raise ValueError("--pimsim-time-ms must be positive")
if args.batch_size <= 0:
parser.error("--batch-size must be positive")
raise ValueError("--batch-size must be positive")
if args.timeout_seconds < 0:
parser.error("--timeout-seconds must be non-negative")
raise ValueError("--timeout-seconds must be non-negative")
comparisons_by_arch: dict[str, tuple[tuple[str, int | str, str], ...]] = {}
configs_by_arch: dict[str, dict[str, Path]] = {}
for arch in args.archs:
@@ -697,7 +647,7 @@ def main() -> int:
and (args.pipeline is None or comparison[1] == args.pipeline)
)
if not comparisons:
parser.error("--pipeline does not belong to the selected --mode")
raise ValueError("--pipeline does not belong to the selected --mode")
configs = {
mode: config_path(arch, mode, args.pimsim_time_ms, write=not args.dry_run)
for mode, _, _ in comparisons
@@ -716,7 +666,7 @@ def main() -> int:
if not MODELS[name].exists()
]
if missing:
parser.error(f"missing model(s): {', '.join(missing)}")
raise ValueError(f"missing model(s): {', '.join(missing)}")
if args.only is not None:
missing_reuse = []
@@ -730,7 +680,7 @@ def main() -> int:
if not required.exists():
missing_reuse.append(str(required))
if missing_reuse:
parser.error(
raise ValueError(
f"--only {args.only} needs existing comparison artifacts: "
+ ", ".join(missing_reuse)
)
@@ -740,41 +690,6 @@ def main() -> int:
if out_dir is not None and not args.dry_run:
out_dir.mkdir(parents=True, exist_ok=True)
print(Style.BRIGHT + f"Found {len(args.models)} Pimcomp model(s) to compare." + Style.RESET_ALL)
print(f"Architectures: {', '.join(args.archs)}")
print(f"Modes: {', '.join(args.mode)}")
print(f"Throughput Pimsim time: {args.pimsim_time_ms} ms")
print(f"Max parallel jobs: {args.jobs}")
print(f"Shared reference batch: {reference_batch_size}")
print(
f"Comparison jobs: "
f"{sum(len(args.models) * len(comparisons) for comparisons in comparisons_by_arch.values())}"
)
print(f"Artifacts root: {out_dir or SUITE}")
print("=" * 72)
print_stage("Prepare shared artifacts", STAGE_COLORS["Build runner"])
try:
for name in args.models:
try:
run(
prepare_common_command(
MODELS[name],
common_dir(out_dir, name, common_root),
args.timeout_seconds,
reference_batch_size,
args.jobs,
),
dry_run=args.dry_run,
)
finally:
print_completed(f"shared artifacts: {name}")
except KeyboardInterrupt:
remove_lock_files(out_dir or SUITE)
print("Interrupted; cleaned validation lock files.", file=sys.stderr)
return 130
failed = []
comparison_specs: list[ComparisonSpec] = []
shared_pimcomp_by_group: dict[tuple[str, str, str], Path] = {}
for arch in args.archs:
@@ -808,117 +723,171 @@ def main() -> int:
)
)
if args.dry_run:
for spec in comparison_specs:
command = comparison_command_for(
spec,
args,
reuse_shared_pimcomp=not spec.anchor,
)
if run(command, dry_run=True, check=False):
return SuitePlan(args, out_dir, common_root, comparisons_by_arch,
reference_batch_size, comparison_specs, {}, {})
def execution_key(plan: SuitePlan, spec: ComparisonSpec) -> tuple:
args = plan.args
return (spec.model, spec.common_dir, spec.config, spec.mode, spec.pipeline,
tuple(args.raptor_extra_args), args.batch_size, args.no_fast,
args.threads_per_job)
def build_tasks(plans: list[SuitePlan]) -> list[ExperimentTask]:
tasks = []
references = {}
comparison_sources = {execution_key(plan, spec): spec.output_dir
for plan in plans if not plan.args.ablation_variant and plan.args.only is None
and not plan.args.raptor_only for spec in plan.specs}
for plan in plans:
for name in plan.args.models:
common = common_dir(plan.root, name, plan.common_root)
previous = references.get(common)
if previous is None or previous[0] < plan.reference_batch_size:
references[common] = (plan.reference_batch_size, name, plan.args)
for common, (batch_size, name, args) in references.items():
command = prepare_common_command(MODELS[name], common, args.timeout_seconds,
batch_size, args.threads_per_job)
command += ["--threads-per-job", str(args.threads_per_job)]
tasks.append(ExperimentTask("reference:" + str(common), lambda command=command: command,
(), common / "prepare.log"))
for plan in plans:
args = plan.args
for spec in plan.specs:
source = comparison_sources.get(execution_key(plan, spec)) if args.ablation_variant == "baseline" else None
report_root = source if source is not None else spec.output_dir
plan.previous_reports[spec.output_dir] = pimcomp_report_stamp(report_root / "pimcomp/output")
if args.ablation_variant == "baseline" and execution_key(plan, spec) in comparison_sources:
plan.aliases[spec.output_dir] = comparison_sources[execution_key(plan, spec)]
continue
reuse = not (spec.anchor or args.only == "raptor" or args.raptor_only)
dependencies = ["reference:" + str(spec.common_dir)]
previous_report = pimcomp_report_stamp(spec.shared_pimcomp_dir) if reuse else None
if reuse:
dependencies.append(str(spec.shared_pimcomp_dir.parent.parent))
def command(spec=spec, args=args, reuse=reuse, previous=previous_report):
if reuse and not args.dry_run and not pimcomp_report_ready(spec.shared_pimcomp_dir, previous):
raise ValueError(f"Pimcomp prerequisite produced no fresh report: {spec.shared_pimcomp_dir}")
return comparison_command_for(spec, args, reuse_shared_pimcomp=reuse)
tasks.append(ExperimentTask(str(spec.output_dir), command, tuple(dependencies),
spec.output_dir / "comparison.log"))
return tasks
def report_for(plan: SuitePlan, spec: ComparisonSpec) -> Path:
return plan.aliases.get(spec.output_dir, spec.output_dir) / "pimcomp/comparison_report.json"
def finish_plan(plan: SuitePlan, results: dict[str, TaskResult]) -> int:
args = plan.args
failed, diagnostic = [], []
overrides = {}
for spec in plan.specs:
directory = plan.aliases.get(spec.output_dir, spec.output_dir)
result = results.get(str(directory))
fresh = pimcomp_report_ready(directory / "pimcomp/output", plan.previous_reports[spec.output_dir])
overrides[spec.output_dir] = report_for(plan, spec) if result and not result.returncode and fresh else None
for arch in args.archs:
output = write_results_csv(plan.root, arch, args.models, plan.comparisons[arch],
args.ablation_variant, overrides, args.raptor_only)
if plan.aliases:
destination = (plan.root or SUITE) / "ablation" / args.ablation_variant / "source_reports.json"
destination.write_text(json.dumps({str(k): str(v) if v is not None else None
for k, v in overrides.items()}, indent=2) + "\n")
for spec in plan.specs:
result = results.get(str(plan.aliases.get(spec.output_dir, spec.output_dir)))
path = report_for(plan, spec)
report = json.loads(path.read_text()) if overrides[spec.output_dir] is not None else {}
if result is None or result.returncode or overrides[spec.output_dir] is None:
failed.append(spec.label)
elif not comparison_passed(report):
if args.ablation_variant == "no-sync" and completed_synchronization_ablation(report):
diagnostic.append(spec.label)
else:
failed.append(spec.label)
elif comparison_specs:
raptor_only_run = args.only == "raptor" or args.raptor_only
anchor_specs = comparison_specs if raptor_only_run else [
spec for spec in comparison_specs if spec.anchor
]
anchor_jobs = [
(
spec.label,
comparison_command_for(spec, args, reuse_shared_pimcomp=False),
None,
)
for spec in anchor_specs
]
dependent_specs = [] if raptor_only_run else [
spec for spec in comparison_specs if not spec.anchor
]
print_directly = min(args.jobs, len(comparison_specs)) == 1
try:
with (nullcontext(None) if print_directly else TemporaryDirectory(prefix="raptor-pimcomp-")) as log_dir:
previous_reports = {
spec.shared_pimcomp_dir: pimcomp_report_stamp(spec.shared_pimcomp_dir)
for spec in anchor_specs
}
anchor_failed, log_offset = run_comparison_jobs(
anchor_jobs,
args.jobs,
Path(log_dir) if log_dir else None,
0,
)
failed.extend(anchor_failed)
dependent_jobs = [
(
spec.label,
comparison_command_for(
spec,
args,
reuse_shared_pimcomp=pimcomp_report_ready(
spec.shared_pimcomp_dir, previous_reports[spec.shared_pimcomp_dir]
),
),
None,
)
for spec in dependent_specs
]
dependent_failed, _ = run_comparison_jobs(
dependent_jobs,
args.jobs,
Path(log_dir) if log_dir else None,
log_offset,
)
failed.extend(dependent_failed)
except KeyboardInterrupt:
remove_lock_files(out_dir or SUITE)
print("Interrupted; cleaned validation lock files.", file=sys.stderr)
return 130
if args.dry_run:
return 1 if failed else 0
remove_lock_files(out_dir or SUITE)
results_path = None
for arch in args.archs:
results_path = write_results_csv(
out_dir,
arch,
args.models,
comparisons_by_arch[arch],
args.ablation_variant,
)
assert results_path is not None
print_stage(results_path.name, STAGE_COLORS["Compare outputs"])
print(results_path.read_text(encoding="utf-8"), end="")
diagnostic_failures = []
for arch in args.archs:
for name in args.models:
for mode, pipeline, _ in comparisons_by_arch[arch]:
label = f"{arch}/{name}/{mode}/pipeline{pipeline}"
report_path = result_dir(
out_dir, name, arch, mode, pipeline, args.ablation_variant
) / "pimcomp/comparison_report.json"
report = json.loads(report_path.read_text(encoding="utf-8")) if report_path.exists() else {}
if not comparison_passed(report):
if (label not in failed and args.ablation_variant == "no-sync"
and completed_synchronization_ablation(report)):
diagnostic_failures.append(label)
elif label not in failed:
failed.append(label)
print(Style.BRIGHT + Fore.CYAN + "Summary" + Style.RESET_ALL)
total_jobs = sum(len(args.models) * len(comparisons) for comparisons in comparisons_by_arch.values())
print(Style.BRIGHT + f"Passed: {total_jobs - len(failed) - len(diagnostic_failures)}" + Style.RESET_ALL)
print(Style.BRIGHT + f"Failed: {len(failed)}" + Style.RESET_ALL)
if diagnostic_failures:
print("No-sync numerical failures (diagnostic only): " + ", ".join(diagnostic_failures))
print(Style.BRIGHT + f"Results: {results_path}" + Style.RESET_ALL)
print(f"Results: {output}; passed: {len(plan.specs) - len(failed) - len(diagnostic)}; "
f"failed: {len(failed)}; diagnostic no-sync failures: {len(diagnostic)}", flush=True)
if failed:
print(
Fore.RED + f"Failed comparisons: {', '.join(failed)}" + Style.RESET_ALL,
file=sys.stderr,
)
return 1
return 0
print("Failed comparisons: " + ", ".join(failed), file=sys.stderr)
remove_lock_files(plan.root or SUITE)
return int(bool(failed))
def run_plans(plans: list[SuitePlan]) -> int:
tasks = build_tasks(plans)
args = plans[0].args
if any((plan.args.jobs, plan.args.threads_per_job) != (args.jobs, args.threads_per_job) for plan in plans):
raise ValueError("all experiment plans must use the same job and thread limits")
if args.dry_run:
for task in tasks:
run(task.command(), dry_run=True, check=False)
return 0
print(f"Concurrent jobs: {args.jobs}; threads per job: {args.threads_per_job}", flush=True)
results = run_tasks(tasks, args.jobs, args.threads_per_job, REPO)
# Only the parent writes aggregate CSVs.
failures = [finish_plan(plan, results) for plan in plans]
return int(any(failures))
def combined_ablation_plans(args: argparse.Namespace) -> tuple[list[SuitePlan], object, Path]:
sys.path.insert(0, str(REPO / "validation/tools/pim/ablation"))
import run_ablation as ablation
from copy import deepcopy
root = args.ablation_out_dir or args.out_dir or SUITE
models = [name for name in args.models if name in ablation.ABLATION_DEFAULT_MODELS]
if not models:
raise ValueError("combined ablation needs at least one of the four ablation models")
cases = [(mode, depth) for mode, depth, _ in COMPARISONS
if mode in args.mode and (depth in (1, 4) if args.pipeline is None else depth == args.pipeline)]
if args.pipeline is not None and not cases:
cases = [("throughput", args.pipeline)]
plans = []
for variant in ablation.VARIANTS:
for mode, depth in cases:
if variant.name == "no-sync" and mode != "throughput":
continue
selected = deepcopy(args)
selected.out_dir, selected.models = root, models
selected.common_dir = args.common_dir or args.out_dir or SUITE
selected.raptor_only, selected.only = True, None
selected.ablation_variant = variant.name
selected.mode, selected.pipeline = [mode], depth
selected.raptor_extra_args += list(variant.raptor_args)
plans.append(make_plan(selected))
return plans, ablation, root
def main(argv: list[str] | None = None) -> int:
args = parse_args(argv)
if args.clean:
print(f"Removed {clean_artifacts(args.out_dir, args.models, args.common_dir)} comparison artifact path(s).")
return 0
try:
plans = [make_plan(args)]
if args.with_ablation:
if args.only or args.raptor_only or args.ablation_variant:
raise ValueError("--with-ablation requires a full compiler comparison")
extra, ablation, ablation_root = combined_ablation_plans(args)
plans.extend(extra)
result = run_plans(plans)
if args.with_ablation and not args.dry_run:
cases = frozenset((spec.mode, str(spec.pipeline)) for plan in extra for spec in plan.specs)
variants = [variant for variant in ablation.VARIANTS
if any(plan.args.ablation_variant == variant.name for plan in extra)]
_, failures = ablation.aggregate_results(ablation_root, variants, cases,
frozenset(extra[0].args.models),
output=ablation.DEFAULT_RESULTS_PATH if args.out_dir is None and args.ablation_out_dir is None else None)
result = int(bool(result or failures))
return result
except KeyboardInterrupt:
print("Interrupted; terminated experiment workers.", file=sys.stderr)
return 130
except ValueError as exc:
print(str(exc), file=sys.stderr)
return 2
if __name__ == "__main__":