Files
Niccolò Nicolosi 66db4b66da
Validate Operations / validate-operations (push) Has been cancelled
add paper artifact
2026-11-03 14:37:25 +01:00

154 lines
5.7 KiB
Python

"""Run experiment subprocesses as their dependencies complete."""
from __future__ import annotations
import argparse
from collections import deque
from dataclasses import dataclass
import json
import os
from pathlib import Path
import signal
import subprocess
import time
from typing import Callable
def available_processors() -> int:
if hasattr(os, "sched_getaffinity"):
return len(os.sched_getaffinity(0))
return os.cpu_count() or 1
def positive_integer(value: str) -> int:
number = int(value)
if number < 1:
raise argparse.ArgumentTypeError("must be positive")
return number
def thread_environment(threads: int) -> dict[str, str]:
return {**os.environ, "OMP_NUM_THREADS": str(threads),
"OMP_THREAD_LIMIT": str(threads), "OMP_DYNAMIC": "FALSE",
"OMP_WAIT_POLICY": "PASSIVE", "OPENBLAS_NUM_THREADS": str(threads),
"MKL_NUM_THREADS": str(threads), "RAYON_NUM_THREADS": str(threads),
"CMAKE_BUILD_PARALLEL_LEVEL": str(threads), "CARGO_BUILD_JOBS": str(threads)}
@dataclass(frozen=True)
class ExperimentTask:
key: str
command: Callable[[], list[str]]
dependencies: tuple[str, ...]
log_path: Path
@dataclass(frozen=True)
class TaskResult:
returncode: int
error: str = ""
def stop_processes(processes: list[subprocess.Popen]) -> None:
for process in processes:
try:
os.killpg(process.pid, signal.SIGTERM)
except ProcessLookupError:
pass
deadline = time.monotonic() + 5
for process in processes:
try:
process.wait(timeout=max(0, deadline - time.monotonic()))
except subprocess.TimeoutExpired:
try:
os.killpg(process.pid, signal.SIGKILL)
except ProcessLookupError:
pass
process.wait()
def run_tasks(tasks: list[ExperimentTask], jobs: int, threads: int,
cwd: Path) -> dict[str, TaskResult]:
if jobs < 1 or threads < 1:
raise ValueError("jobs and threads-per-job must be positive")
by_key = {task.key: task for task in tasks}
if len(by_key) != len(tasks):
raise ValueError("duplicate experiment task")
waiting = {task.key: set(task.dependencies) for task in tasks}
children = {key: [] for key in by_key}
for task in tasks:
for dependency in task.dependencies:
if dependency not in by_key:
raise ValueError(f"unknown dependency: {dependency}")
children[dependency].append(task.key)
ready = deque(key for key in by_key if not waiting[key])
degrees = {key: len(dependencies) for key, dependencies in waiting.items()}
pending = deque(ready)
visited = 0
while pending:
visited += 1
for child in children[pending.popleft()]:
degrees[child] -= 1
if not degrees[child]:
pending.append(child)
if visited != len(tasks):
raise ValueError("experiment dependency cycle")
active = {}
results = {}
records = {}
environment = thread_environment(threads)
def complete(key: str, result: TaskResult) -> None:
results[key] = result
task = by_key[key]
record = records.get(key, {"dependencies": task.dependencies, "threads_per_job": threads})
record.update(returncode=result.returncode, error=result.error)
task.log_path.parent.mkdir(parents=True, exist_ok=True)
task.log_path.with_suffix(".execution.json").write_text(json.dumps(record, indent=2) + "\n")
print(f"[{'Failed' if result.returncode else 'Completed'}] {key}"
f"{': ' + result.error if result.error else ''}", flush=True)
for child in children[key]:
waiting[child].remove(key)
if not waiting[child]:
ready.append(child)
try:
while ready or active:
while ready and len(active) < jobs:
key = ready.popleft()
task = by_key[key]
failed = [dep for dep in task.dependencies if results[dep].returncode]
if failed:
complete(key, TaskResult(1, "prerequisite failed: " + ", ".join(failed)))
continue
log = None
try:
command = task.command()
task.log_path.parent.mkdir(parents=True, exist_ok=True)
log = task.log_path.open("wb")
metadata = {"command": command, "threads_per_job": threads,
"dependencies": task.dependencies}
records[key] = metadata
task.log_path.with_suffix(".execution.json").write_text(json.dumps(metadata, indent=2) + "\n")
process = subprocess.Popen(command, cwd=cwd, env=environment,
stdout=log, stderr=subprocess.STDOUT, start_new_session=True)
except (OSError, ValueError, RuntimeError) as exc:
if log is not None:
log.close()
complete(key, TaskResult(1, str(exc)))
continue
active[key] = process, log
print(f"[Started] {key}; log: {task.log_path}", flush=True)
completed = [(key, process.returncode) for key, (process, _) in active.items()
if process.poll() is not None]
for key, returncode in completed:
_, log = active.pop(key)
log.close()
complete(key, TaskResult(returncode))
if active and not completed:
time.sleep(.05)
return results
finally:
stop_processes([process for process, _ in active.values()])
for _, log in active.values():
log.close()