154 lines
5.7 KiB
Python
154 lines
5.7 KiB
Python
"""Run experiment subprocesses as their dependencies complete."""
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
from collections import deque
|
|
from dataclasses import dataclass
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import signal
|
|
import subprocess
|
|
import time
|
|
from typing import Callable
|
|
|
|
|
|
def available_processors() -> int:
|
|
if hasattr(os, "sched_getaffinity"):
|
|
return len(os.sched_getaffinity(0))
|
|
return os.cpu_count() or 1
|
|
|
|
|
|
def positive_integer(value: str) -> int:
|
|
number = int(value)
|
|
if number < 1:
|
|
raise argparse.ArgumentTypeError("must be positive")
|
|
return number
|
|
|
|
|
|
def thread_environment(threads: int) -> dict[str, str]:
|
|
return {**os.environ, "OMP_NUM_THREADS": str(threads),
|
|
"OMP_THREAD_LIMIT": str(threads), "OMP_DYNAMIC": "FALSE",
|
|
"OMP_WAIT_POLICY": "PASSIVE", "OPENBLAS_NUM_THREADS": str(threads),
|
|
"MKL_NUM_THREADS": str(threads), "RAYON_NUM_THREADS": str(threads),
|
|
"CMAKE_BUILD_PARALLEL_LEVEL": str(threads), "CARGO_BUILD_JOBS": str(threads)}
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ExperimentTask:
|
|
key: str
|
|
command: Callable[[], list[str]]
|
|
dependencies: tuple[str, ...]
|
|
log_path: Path
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class TaskResult:
|
|
returncode: int
|
|
error: str = ""
|
|
|
|
|
|
def stop_processes(processes: list[subprocess.Popen]) -> None:
|
|
for process in processes:
|
|
try:
|
|
os.killpg(process.pid, signal.SIGTERM)
|
|
except ProcessLookupError:
|
|
pass
|
|
deadline = time.monotonic() + 5
|
|
for process in processes:
|
|
try:
|
|
process.wait(timeout=max(0, deadline - time.monotonic()))
|
|
except subprocess.TimeoutExpired:
|
|
try:
|
|
os.killpg(process.pid, signal.SIGKILL)
|
|
except ProcessLookupError:
|
|
pass
|
|
process.wait()
|
|
|
|
|
|
def run_tasks(tasks: list[ExperimentTask], jobs: int, threads: int,
|
|
cwd: Path) -> dict[str, TaskResult]:
|
|
if jobs < 1 or threads < 1:
|
|
raise ValueError("jobs and threads-per-job must be positive")
|
|
by_key = {task.key: task for task in tasks}
|
|
if len(by_key) != len(tasks):
|
|
raise ValueError("duplicate experiment task")
|
|
waiting = {task.key: set(task.dependencies) for task in tasks}
|
|
children = {key: [] for key in by_key}
|
|
for task in tasks:
|
|
for dependency in task.dependencies:
|
|
if dependency not in by_key:
|
|
raise ValueError(f"unknown dependency: {dependency}")
|
|
children[dependency].append(task.key)
|
|
ready = deque(key for key in by_key if not waiting[key])
|
|
degrees = {key: len(dependencies) for key, dependencies in waiting.items()}
|
|
pending = deque(ready)
|
|
visited = 0
|
|
while pending:
|
|
visited += 1
|
|
for child in children[pending.popleft()]:
|
|
degrees[child] -= 1
|
|
if not degrees[child]:
|
|
pending.append(child)
|
|
if visited != len(tasks):
|
|
raise ValueError("experiment dependency cycle")
|
|
active = {}
|
|
results = {}
|
|
records = {}
|
|
environment = thread_environment(threads)
|
|
|
|
def complete(key: str, result: TaskResult) -> None:
|
|
results[key] = result
|
|
task = by_key[key]
|
|
record = records.get(key, {"dependencies": task.dependencies, "threads_per_job": threads})
|
|
record.update(returncode=result.returncode, error=result.error)
|
|
task.log_path.parent.mkdir(parents=True, exist_ok=True)
|
|
task.log_path.with_suffix(".execution.json").write_text(json.dumps(record, indent=2) + "\n")
|
|
print(f"[{'Failed' if result.returncode else 'Completed'}] {key}"
|
|
f"{': ' + result.error if result.error else ''}", flush=True)
|
|
for child in children[key]:
|
|
waiting[child].remove(key)
|
|
if not waiting[child]:
|
|
ready.append(child)
|
|
|
|
try:
|
|
while ready or active:
|
|
while ready and len(active) < jobs:
|
|
key = ready.popleft()
|
|
task = by_key[key]
|
|
failed = [dep for dep in task.dependencies if results[dep].returncode]
|
|
if failed:
|
|
complete(key, TaskResult(1, "prerequisite failed: " + ", ".join(failed)))
|
|
continue
|
|
log = None
|
|
try:
|
|
command = task.command()
|
|
task.log_path.parent.mkdir(parents=True, exist_ok=True)
|
|
log = task.log_path.open("wb")
|
|
metadata = {"command": command, "threads_per_job": threads,
|
|
"dependencies": task.dependencies}
|
|
records[key] = metadata
|
|
task.log_path.with_suffix(".execution.json").write_text(json.dumps(metadata, indent=2) + "\n")
|
|
process = subprocess.Popen(command, cwd=cwd, env=environment,
|
|
stdout=log, stderr=subprocess.STDOUT, start_new_session=True)
|
|
except (OSError, ValueError, RuntimeError) as exc:
|
|
if log is not None:
|
|
log.close()
|
|
complete(key, TaskResult(1, str(exc)))
|
|
continue
|
|
active[key] = process, log
|
|
print(f"[Started] {key}; log: {task.log_path}", flush=True)
|
|
completed = [(key, process.returncode) for key, (process, _) in active.items()
|
|
if process.poll() is not None]
|
|
for key, returncode in completed:
|
|
_, log = active.pop(key)
|
|
log.close()
|
|
complete(key, TaskResult(returncode))
|
|
if active and not completed:
|
|
time.sleep(.05)
|
|
return results
|
|
finally:
|
|
stop_processes([process for process, _ in active.values()])
|
|
for _, log in active.values():
|
|
log.close()
|