Merge remote-tracking branch 'origin/main'
Validate Operations / validate-operations (push) Has been cancelled
Validate Operations / validate-operations (push) Has been cancelled
This commit is contained in:
@@ -9,6 +9,9 @@ Run the synchronization/Spatial-planning ablation from the repository root:
|
||||
Defaults: `vgg8 resnet18 resnet34 googlenet`, `arch-a arch-b`, latency
|
||||
(pipeline 1), and throughput (pipeline 4). Select `yolo11n` or `arch-c`
|
||||
explicitly when needed. Only Raptor runs; Pimcomp validation is `SKIPPED`.
|
||||
Ready models, architectures, modes, and variants run concurrently in one dependency queue.
|
||||
References are prepared once per model. `--jobs` and `--threads-per-job` both default
|
||||
to the available logical processor count; override them to limit CPU contention.
|
||||
|
||||
| Variant | Raptor options |
|
||||
|---|---|
|
||||
@@ -34,7 +37,7 @@ suite pipelines for that mode. Pipeline-only selection uses its matching mode.
|
||||
`--pipeline auto` requests automatic throughput depth selection; positive numeric
|
||||
depths retain their exact-stage meaning.
|
||||
`--only` is rejected because
|
||||
the wrapper always runs Raptor only. Ctrl+C stops the active variant's workers.
|
||||
the wrapper always runs Raptor only. Ctrl+C stops all active experiment workers.
|
||||
|
||||
Artifacts use `<out-dir>/<model>/artifacts/<arch>/<mode>[/pipelineN]/ablation/<variant>/`;
|
||||
reference inputs, outputs, and runners are shared under `artifacts/common/`.
|
||||
|
||||
@@ -6,10 +6,7 @@ from __future__ import annotations
|
||||
import argparse
|
||||
import csv
|
||||
import math
|
||||
import os
|
||||
import shlex
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
@@ -26,6 +23,7 @@ from raptor_validation.pimcomp_models import (
|
||||
)
|
||||
from raptor_validation.artifacts import remove_lock_files
|
||||
from raptor_validation.raptor import parse_pipeline_depth, pipeline_sort_key
|
||||
from raptor_validation.experiment_scheduler import available_processors, positive_integer
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
@@ -79,9 +77,14 @@ def parse_args(argv: list[str] | None = None) -> tuple[argparse.Namespace, list[
|
||||
help="Run only this variant; baseline is added when needed. Repeat as needed.",
|
||||
)
|
||||
parser.add_argument("--dry-run", action="store_true", help="Print runner commands without modifying files.")
|
||||
parser.add_argument("--jobs", "-j", type=positive_integer, default=available_processors(),
|
||||
help="Maximum concurrent cases (default: available logical processors).")
|
||||
parser.add_argument("--threads-per-job", type=positive_integer, default=available_processors(),
|
||||
help="Threads per case (default: available logical processors).")
|
||||
args, forwarded = parser.parse_known_args(argv)
|
||||
if any(option == "--only" or option.startswith("--only=") for option in forwarded):
|
||||
parser.error("--only is not supported; the ablation wrapper always runs Raptor only")
|
||||
forwarded.extend(("--jobs", str(args.jobs), "--threads-per-job", str(args.threads_per_job)))
|
||||
return args, forwarded
|
||||
|
||||
|
||||
@@ -156,23 +159,6 @@ def runner_command(
|
||||
return command
|
||||
|
||||
|
||||
def terminate_process_group(process: subprocess.Popen[bytes]) -> None:
|
||||
if process.poll() is not None:
|
||||
return
|
||||
try:
|
||||
os.killpg(process.pid, signal.SIGTERM)
|
||||
except ProcessLookupError:
|
||||
return
|
||||
try:
|
||||
process.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
try:
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
return
|
||||
process.wait()
|
||||
|
||||
|
||||
def percentage_difference(
|
||||
value: str | None, reference: str | None, *, lower_is_better: bool = False
|
||||
) -> str:
|
||||
@@ -313,33 +299,27 @@ def main(argv: list[str] | None = None) -> int:
|
||||
)
|
||||
return 0
|
||||
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
failed = []
|
||||
current_process = None
|
||||
sys.path.insert(0, str(RUNNER.parent))
|
||||
import run_pimcomp_models as suite
|
||||
plans = []
|
||||
try:
|
||||
for variant in variants:
|
||||
for case_label, case_forwarded in case_sets:
|
||||
for _, case_forwarded in case_sets:
|
||||
if variant.name == "no-sync" and case_key(case_forwarded)[0] != "throughput":
|
||||
continue
|
||||
command = runner_command(
|
||||
out_dir, variant, args.models, case_forwarded, out_dir, False
|
||||
)
|
||||
print(f"[{variant.name}/{case_label}] {shlex.join(command)}")
|
||||
current_process = subprocess.Popen(
|
||||
command,
|
||||
cwd=REPO,
|
||||
start_new_session=True,
|
||||
)
|
||||
returncode = current_process.wait()
|
||||
current_process = None
|
||||
if returncode:
|
||||
failed.append(f"{variant.name}/{case_label}: runner exited with {returncode}")
|
||||
command = runner_command(out_dir, variant, args.models, case_forwarded, out_dir, False)
|
||||
selected = suite.parse_args(command[2:])
|
||||
if selected.clean:
|
||||
print(f"Removed {suite.clean_artifacts(out_dir, args.models, out_dir)} artifact path(s).")
|
||||
return 0
|
||||
plans.append(suite.make_plan(selected))
|
||||
failed = ["experiment jobs failed"] if suite.run_plans(plans) else []
|
||||
except KeyboardInterrupt:
|
||||
if current_process is not None:
|
||||
terminate_process_group(current_process)
|
||||
remove_lock_files(out_dir)
|
||||
print("Interrupted; terminated the active ablation job.", file=sys.stderr)
|
||||
print("Interrupted; terminated ablation workers.", file=sys.stderr)
|
||||
return 130
|
||||
except ValueError as exc:
|
||||
print(str(exc), file=sys.stderr)
|
||||
return 2
|
||||
|
||||
remove_lock_files(out_dir)
|
||||
selected_cases = frozenset(case_key(arguments) for _, arguments in case_sets)
|
||||
|
||||
@@ -11,7 +11,12 @@ built release Raptor, Pimcomp, and `pimsim-nn`.
|
||||
```
|
||||
|
||||
Defaults: all five models, Arch-A/B, latency pipeline 1, throughput pipelines
|
||||
2/4/8, four workers, and a functional throughput batch of 64.
|
||||
2/4/8 and a functional throughput batch of 64.
|
||||
Both `--jobs` and `--threads-per-job` default to the available logical processor count.
|
||||
Their limits multiply: 40 jobs with 40 threads can create 1,600 threads.
|
||||
Use `--jobs 8 --threads-per-job 4` to reduce contention.
|
||||
The thread limit controls OpenMP, Raptor's compilation thread pool (`-j`), BLAS, and Rayon.
|
||||
Reference batches use that many workers, with one thread per reference process.
|
||||
Use `--help` for the complete option reference. Numeric depths greater than one
|
||||
require exactly that many active compute stages. `--pipeline auto` requests
|
||||
throughput depth selection. Reports include `raptor_selected_pipeline`; the
|
||||
@@ -26,6 +31,8 @@ requested depth remains in the existing pipeline field.
|
||||
| `--only {raptor,pimcomp}` | Rerun one compiler and reuse the other's existing report. |
|
||||
| `--raptor-only` | Run Raptor without invoking Pimcomp. |
|
||||
| `--batch-size N`, `--jobs N` | Set functional throughput batch size and worker count. |
|
||||
| `--threads-per-job N` | Set the thread limit inside each job. |
|
||||
| `--with-ablation`, `--ablation-out-dir PATH` | Schedule both suites together; optionally select a separate ablation root. |
|
||||
| `--pimsim-time-ms N` | Throughput convergence deadline; default: 1000 ms. |
|
||||
| `--no-fast` | Use full-duration throughput simulation for authoritative experiments. |
|
||||
| `--timeout-seconds N` | Per-stage timeout; default: 0 (unlimited). |
|
||||
@@ -45,11 +52,19 @@ Artifacts use
|
||||
under `artifacts/common/`
|
||||
and `comparison_report.{md,json}` under each comparison's `pimcomp/`.
|
||||
YOLO and GoogLeNet use `artifacts/common-full-model/` to avoid reusing historical Softmax-free references.
|
||||
References are prepared once per model. Each case starts when its own prerequisites finish.
|
||||
Pimcomp runs once per model/architecture/mode; later pipelines reuse its newly
|
||||
written report, including failures without simulation artifacts. Missing or
|
||||
unchanged anchor reports trigger fresh attempts. The runner continues after failures and exits nonzero if a
|
||||
unchanged prerequisite reports skip dependent cases without repeating Pimcomp work.
|
||||
Skipped cases retain unavailable metrics rather than old measurements.
|
||||
The runner continues after failures and exits nonzero if a
|
||||
compiler, simulation, or functional comparison fails.
|
||||
|
||||
Combined runs share references with ablation and reuse matching Raptor baseline reports.
|
||||
`ablation/baseline/source_reports.json` records those report locations.
|
||||
Only the parent process writes aggregate CSVs. Per-case logs and execution settings remain beside each case.
|
||||
Thread limits are recorded in reports and checked during report reuse.
|
||||
|
||||
If fast-mode convergence fails, the reports and CSV retain valid full-duration
|
||||
fallback throughput and output interval values. This is not a converged steady-state result.
|
||||
Per-case JSON records `throughput_measurement: full_duration_fallback` and
|
||||
|
||||
@@ -46,6 +46,7 @@ from raptor_validation.onnx_utils import (
|
||||
)
|
||||
from raptor_validation.raptor import compile_with_raptor, read_selected_pipeline_depth
|
||||
from raptor_validation.output_comparison import VALIDATION_POLICY, compare_output_arrays
|
||||
from raptor_validation.experiment_scheduler import available_processors, positive_integer, thread_environment
|
||||
from raptor_validation.pimsim_nn import (
|
||||
export_raptor_pimsim_artifact,
|
||||
parse_pimsim_nn_metrics,
|
||||
@@ -156,6 +157,7 @@ def run_logged(
|
||||
steps: list[StepRecord],
|
||||
stage: str | None = None,
|
||||
print_header: bool = True,
|
||||
env: dict[str, str] | None = None,
|
||||
) -> str:
|
||||
if print_header:
|
||||
print_step(name, cmd, cwd, stage)
|
||||
@@ -172,6 +174,7 @@ def run_logged(
|
||||
stderr=subprocess.STDOUT,
|
||||
text=True,
|
||||
timeout=None if timeout_sec == 0 else timeout_sec,
|
||||
env=env,
|
||||
)
|
||||
except subprocess.TimeoutExpired as exc:
|
||||
duration = time.perf_counter() - start
|
||||
@@ -364,7 +367,8 @@ def compile_reference(
|
||||
run_logged(
|
||||
"Compile reference ONNX IR",
|
||||
[str(args.raptor_path), str(model_path), "-o", str(onnx_ir_base), "--EmitONNXIR",
|
||||
"--mlir-elide-elementsattrs-if-larger=16", "--enable-conv-opt-pass=false"],
|
||||
"--mlir-elide-elementsattrs-if-larger=16", "--enable-conv-opt-pass=false",
|
||||
f"-j={args.threads_per_job}"],
|
||||
cwd=REPO,
|
||||
timeout_sec=args.timeout_seconds,
|
||||
steps=steps,
|
||||
@@ -372,7 +376,7 @@ def compile_reference(
|
||||
)
|
||||
run_logged(
|
||||
"Compile reference native",
|
||||
[str(args.raptor_path), "-O3", str(model_path), "-o", str(runner_base)],
|
||||
[str(args.raptor_path), "-O3", str(model_path), "-o", str(runner_base), f"-j={args.threads_per_job}"],
|
||||
cwd=REPO,
|
||||
timeout_sec=args.timeout_seconds,
|
||||
steps=steps,
|
||||
@@ -400,7 +404,7 @@ def compile_reference(
|
||||
)
|
||||
run_logged(
|
||||
"Build runner",
|
||||
["cmake", "--build", ".", "-j"],
|
||||
["cmake", "--build", ".", "--parallel", str(args.threads_per_job)],
|
||||
cwd=build_dir,
|
||||
timeout_sec=args.timeout_seconds,
|
||||
steps=steps,
|
||||
@@ -446,6 +450,7 @@ def generate_reference_outputs(
|
||||
timeout_sec=args.timeout_seconds,
|
||||
steps=steps,
|
||||
print_header=print_header,
|
||||
env=thread_environment(1),
|
||||
)
|
||||
return reference_dir
|
||||
|
||||
@@ -576,6 +581,7 @@ def compile_raptor_target(
|
||||
f"--core-count={hardware['core_count']}",
|
||||
f"--pim-target-config={args.pimcomp_config}",
|
||||
"--pim-emit-json",
|
||||
f"-j={args.threads_per_job}",
|
||||
*args.raptor_extra_arg,
|
||||
]
|
||||
print_step("Compile Raptor", cmd, REPO, "Compile Pim")
|
||||
@@ -584,6 +590,7 @@ def compile_raptor_target(
|
||||
raptor_extra_args = [
|
||||
f"--pim-target-config={args.pimcomp_config}",
|
||||
"--pim-emit-json",
|
||||
f"-j={args.threads_per_job}",
|
||||
*args.raptor_extra_arg,
|
||||
]
|
||||
try:
|
||||
@@ -1202,6 +1209,7 @@ def experiment_context(args, model, functional_model, hardware, side):
|
||||
"oracle": args.pimsim_nn_build_dir / "ChipTest",
|
||||
"comparison": Path(__file__),
|
||||
"validation": VALIDATION_DIR / "raptor_validation/output_comparison.py",
|
||||
"thread_policy": VALIDATION_DIR / "raptor_validation/experiment_scheduler.py",
|
||||
"functional_simulator": args.pim_simulator_dir / "target/release/pim-simulator"}
|
||||
if side == "raptor":
|
||||
files["compiler"] = args.raptor_path
|
||||
@@ -1216,6 +1224,7 @@ def experiment_context(args, model, functional_model, hardware, side):
|
||||
settings.update({name: getattr(args, name) for name in (
|
||||
("raptor_extra_arg",) if side == "raptor" else
|
||||
("pimcomp_pipeline", "pimcomp_replication", "pimcomp_model_name"))})
|
||||
settings["threads_per_job"] = getattr(args, "threads_per_job", available_processors())
|
||||
return {"version": 2, "validation_policy": VALIDATION_POLICY,
|
||||
"files": fingerprint_files(files), "hardware": hardware, "settings": settings}
|
||||
|
||||
@@ -1569,9 +1578,9 @@ def main():
|
||||
parser.add_argument(
|
||||
"-j",
|
||||
"--jobs",
|
||||
type=int,
|
||||
default=4,
|
||||
help="Maximum parallel native reference runner processes (default: 4).",
|
||||
type=positive_integer,
|
||||
default=available_processors(),
|
||||
help="Maximum parallel native reference runner processes (default: logical processors).",
|
||||
)
|
||||
parser.add_argument("--pimcomp-pipeline", choices=["element", "batch"])
|
||||
parser.add_argument("--pimcomp-model-name", help="Use a Pimcomp built-in model name such as vgg16.")
|
||||
@@ -1613,7 +1622,10 @@ def main():
|
||||
action="store_true",
|
||||
help="Return a non-zero status if a stage or semantic validation fails.",
|
||||
)
|
||||
parser.add_argument("--threads-per-job", type=positive_integer, default=available_processors(),
|
||||
help="Threads for OpenMP, MLIR compilation, and simulation (default: logical processors).")
|
||||
args = parser.parse_args()
|
||||
os.environ.update(thread_environment(args.threads_per_job))
|
||||
if args.raptor_only and any(
|
||||
option is not None
|
||||
for option in (
|
||||
@@ -2152,6 +2164,7 @@ def main():
|
||||
"hardware": hardware,
|
||||
"pimsim_mode": args.pimsim_mode,
|
||||
"pimsim_time_ms": args.pimsim_time_ms,
|
||||
"threads_per_job": args.threads_per_job,
|
||||
"pimcomp_pipeline": args.pimcomp_pipeline,
|
||||
"pimcomp_replication": args.pimcomp_replication,
|
||||
"pimcomp_model_source": "not_run" if args.raptor_only else "original_onnx",
|
||||
|
||||
@@ -5,16 +5,12 @@ import argparse
|
||||
import csv
|
||||
import json
|
||||
import math
|
||||
import os
|
||||
import shlex
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
from concurrent.futures import ProcessPoolExecutor, as_completed
|
||||
from contextlib import nullcontext, redirect_stderr, redirect_stdout
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from tempfile import TemporaryDirectory
|
||||
|
||||
from colorama import Fore, Style
|
||||
|
||||
@@ -31,7 +27,9 @@ from raptor_validation.artifacts import artifacts_dir, remove_lock_files
|
||||
from raptor_validation.pimsim_nn import parse_pimsim_nn_metrics
|
||||
from raptor_validation.output_comparison import VALIDATION_POLICY
|
||||
from raptor_validation.raptor import parse_pipeline_depth, pipeline_sort_key
|
||||
from raptor_validation.validate_one import STAGE_COLORS
|
||||
from raptor_validation.experiment_scheduler import (
|
||||
ExperimentTask, TaskResult, available_processors, positive_integer, run_tasks,
|
||||
)
|
||||
|
||||
PIMCOMP_SOURCE = REPO / "third_party/PIMCOMP-NN"
|
||||
PIMCOMP_CONFIGS = REPO / "validation/pimsim_configs/pimcomp"
|
||||
@@ -130,6 +128,8 @@ def write_results_csv(
|
||||
models: list[str],
|
||||
comparisons: tuple[tuple[str, int, str], ...] = COMPARISONS,
|
||||
ablation_variant: str | None = None,
|
||||
report_paths: dict[Path, Path | None] | None = None,
|
||||
raptor_only: bool = False,
|
||||
) -> Path:
|
||||
output = DEFAULT_RESULTS_PATH if root is None else root / RESULTS_FILENAME
|
||||
if ablation_variant is not None:
|
||||
@@ -179,6 +179,9 @@ def write_results_csv(
|
||||
report_path = result_dir(
|
||||
root, name, arch, comparison_mode, pipeline, ablation_variant
|
||||
) / "pimcomp/comparison_report.json"
|
||||
if report_paths is not None:
|
||||
directory = result_dir(root, name, arch, comparison_mode, pipeline, ablation_variant)
|
||||
report_path = report_paths.get(directory, report_path)
|
||||
row = {
|
||||
"model": name,
|
||||
"arch": arch,
|
||||
@@ -198,12 +201,12 @@ def write_results_csv(
|
||||
"better_compiler": "NA",
|
||||
"speedup": "NA",
|
||||
}
|
||||
if report_path.exists():
|
||||
if report_path is not None and report_path.exists():
|
||||
report = json.loads(report_path.read_text(encoding="utf-8"))
|
||||
if report.get("validation_policy") != VALIDATION_POLICY:
|
||||
raise ValueError(f"Report uses a different validation policy: {report_path}. Regenerate this comparison.")
|
||||
raptor_values = performance_values(report.get("raptor_performance") or {})
|
||||
pimcomp_values = performance_values(report.get("pimcomp_performance") or {})
|
||||
pimcomp_values = performance_values({} if raptor_only else report.get("pimcomp_performance") or {})
|
||||
raptor_metric = raptor_values["throughput"] if comparison_mode == "throughput" else raptor_values["latency"]
|
||||
pimcomp_metric = pimcomp_values["throughput"] if comparison_mode == "throughput" else pimcomp_values["latency"]
|
||||
row.update(
|
||||
@@ -211,7 +214,7 @@ def write_results_csv(
|
||||
report.get("raptor_validation")
|
||||
),
|
||||
pimcomp_functional_validation=functional_validation_status(
|
||||
report.get("pimcomp_validation")
|
||||
{"status": "skipped"} if raptor_only else report.get("pimcomp_validation")
|
||||
),
|
||||
raptor_throughput_samples_s=format_value(raptor_values["throughput"]),
|
||||
pimcomp_throughput_samples_s=format_value(pimcomp_values["throughput"]),
|
||||
@@ -322,17 +325,6 @@ def format_value(value: float | None) -> str:
|
||||
return f"{value:.6g}"
|
||||
|
||||
|
||||
def print_stage(title: str, color: str) -> None:
|
||||
print(Style.BRIGHT + color + f"[{title}]" + Style.RESET_ALL, flush=True)
|
||||
|
||||
|
||||
def print_completed(label: str, output: str = "") -> None:
|
||||
print(Style.BRIGHT + Fore.CYAN + f"[Completed {label}]" + Style.RESET_ALL)
|
||||
if output:
|
||||
print(output, end="" if output.endswith("\n") else "\n")
|
||||
print("=" * 72, flush=True)
|
||||
|
||||
|
||||
def run(command: list[str], *, dry_run: bool, check: bool = True) -> int:
|
||||
print(f" cwd: {REPO}", flush=True)
|
||||
print(f" $ {shlex.join(command)}", flush=True)
|
||||
@@ -341,29 +333,6 @@ def run(command: list[str], *, dry_run: bool, check: bool = True) -> int:
|
||||
return subprocess.run(command, cwd=REPO, check=check).returncode
|
||||
|
||||
|
||||
def run_comparison_job(job: tuple[str, list[str], Path | None]) -> tuple[str, int, str | None]:
|
||||
label, command, log_path = job
|
||||
sys.stdout.flush()
|
||||
sys.stderr.flush()
|
||||
if log_path is None:
|
||||
return label, run(command, dry_run=False, check=False), None
|
||||
|
||||
saved_stdout = os.dup(1)
|
||||
saved_stderr = os.dup(2)
|
||||
try:
|
||||
with open(log_path, "w", encoding="utf-8", buffering=1) as log:
|
||||
os.dup2(log.fileno(), 1)
|
||||
os.dup2(log.fileno(), 2)
|
||||
with redirect_stdout(log), redirect_stderr(log):
|
||||
returncode = run(command, dry_run=False, check=False)
|
||||
finally:
|
||||
os.dup2(saved_stdout, 1)
|
||||
os.dup2(saved_stderr, 2)
|
||||
os.close(saved_stdout)
|
||||
os.close(saved_stderr)
|
||||
return label, returncode, log_path.read_text(encoding="utf-8", errors="replace")
|
||||
|
||||
|
||||
def validate_pimcomp_source() -> None:
|
||||
header = PIMCOMP_SOURCE / "backend/GeneticAlgorithm.h"
|
||||
source = header.read_text(encoding="utf-8")
|
||||
@@ -469,7 +438,7 @@ def comparison_command_for(
|
||||
reuse_pimcomp_report = report
|
||||
elif reuse_shared_pimcomp:
|
||||
reuse_pimcomp_report = spec.shared_pimcomp_dir.parent / "comparison_report.json"
|
||||
return comparison_command(
|
||||
command = comparison_command(
|
||||
spec.model,
|
||||
spec.functional_model,
|
||||
spec.output_dir,
|
||||
@@ -491,6 +460,7 @@ def comparison_command_for(
|
||||
),
|
||||
reuse_pimcomp_report=reuse_pimcomp_report,
|
||||
)
|
||||
return command + ["--jobs", "1", "--threads-per-job", str(args.threads_per_job)]
|
||||
|
||||
|
||||
def pimcomp_report_stamp(output_dir: Path) -> tuple[int, int, int] | None:
|
||||
@@ -506,40 +476,6 @@ def pimcomp_report_ready(output_dir: Path, previous: tuple[int, int, int] | None
|
||||
return stamp is not None and stamp != previous
|
||||
|
||||
|
||||
def run_comparison_jobs(
|
||||
comparison_jobs: list[tuple[str, list[str], Path | None]],
|
||||
max_workers: int,
|
||||
log_dir: Path | None,
|
||||
log_offset: int,
|
||||
) -> tuple[list[str], int]:
|
||||
if not comparison_jobs:
|
||||
return [], log_offset
|
||||
print_directly = min(max_workers, len(comparison_jobs)) == 1
|
||||
jobs = [
|
||||
(
|
||||
label,
|
||||
command,
|
||||
None if print_directly or log_dir is None else log_dir / f"{log_offset + index}.log",
|
||||
)
|
||||
for index, (label, command, _) in enumerate(comparison_jobs)
|
||||
]
|
||||
with (nullcontext(None) if print_directly else ProcessPoolExecutor(max_workers=max_workers)) as executor:
|
||||
completed = (
|
||||
map(run_comparison_job, jobs)
|
||||
if print_directly
|
||||
else (
|
||||
future.result()
|
||||
for future in as_completed(executor.submit(run_comparison_job, job) for job in jobs)
|
||||
)
|
||||
)
|
||||
failed = []
|
||||
for label, returncode, output in completed:
|
||||
print_completed(label, output or "")
|
||||
if returncode:
|
||||
failed.append(label)
|
||||
return failed, log_offset + len(jobs)
|
||||
|
||||
|
||||
def ensure_throughput_config(arch: str, sim_time_ms: int) -> Path:
|
||||
source = PIMCOMP_CONFIGS / arch / "throughput_config_1000ms.json"
|
||||
target = PIMCOMP_CONFIGS / arch / f"throughput_config_{sim_time_ms}ms.json"
|
||||
@@ -566,7 +502,7 @@ def config_path(arch: str, mode: str, sim_time_ms: int, *, write: bool) -> Path:
|
||||
return path
|
||||
|
||||
|
||||
def main() -> int:
|
||||
def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Compare supported Pimcomp models with Raptor latency and throughput schedules."
|
||||
)
|
||||
@@ -635,9 +571,9 @@ def main() -> int:
|
||||
parser.add_argument(
|
||||
"-j",
|
||||
"--jobs",
|
||||
type=int,
|
||||
default=4,
|
||||
help="Number of comparisons to run in parallel (default: 4).",
|
||||
type=positive_integer,
|
||||
default=available_processors(),
|
||||
help="Maximum concurrent comparisons (default: available logical processors).",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--clean",
|
||||
@@ -657,7 +593,11 @@ def main() -> int:
|
||||
dest="raptor_extra_args",
|
||||
help="Additional argument to pass to Raptor; repeat as needed.",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
parser.add_argument("--threads-per-job", type=positive_integer, default=available_processors(),
|
||||
help="Threads per compiler/simulator job (default: available logical processors).")
|
||||
parser.add_argument("--with-ablation", action="store_true", help="Run comparison and ablation in one dependency queue.")
|
||||
parser.add_argument("--ablation-out-dir", type=Path, help="Output root for a combined ablation run.")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
if args.ablation_variant and Path(args.ablation_variant).name != args.ablation_variant:
|
||||
parser.error("--ablation-variant must be a single directory name")
|
||||
@@ -665,25 +605,35 @@ def main() -> int:
|
||||
if args.only is not None and args.raptor_only:
|
||||
parser.error("--only cannot be combined with --raptor-only")
|
||||
|
||||
return args
|
||||
|
||||
|
||||
@dataclass
|
||||
class SuitePlan:
|
||||
args: argparse.Namespace
|
||||
root: Path | None
|
||||
common_root: Path | None
|
||||
comparisons: dict
|
||||
reference_batch_size: int
|
||||
specs: list[ComparisonSpec]
|
||||
aliases: dict[Path, Path]
|
||||
previous_reports: dict[Path, tuple[int, int, int] | None]
|
||||
|
||||
|
||||
def make_plan(args: argparse.Namespace) -> SuitePlan:
|
||||
out_dir = args.out_dir.resolve() if args.out_dir is not None else None
|
||||
common_root = args.common_dir.resolve() if args.common_dir is not None else None
|
||||
args.archs = list(dict.fromkeys(args.archs))
|
||||
args.models = list(dict.fromkeys(args.models))
|
||||
args.mode = list(dict.fromkeys(args.mode))
|
||||
if args.clean:
|
||||
print(
|
||||
f"Removed {clean_artifacts(out_dir, args.models, common_root)} "
|
||||
"comparison artifact path(s)."
|
||||
)
|
||||
return 0
|
||||
|
||||
if args.jobs < 1:
|
||||
parser.error("--jobs must be at least 1")
|
||||
raise ValueError("--jobs must be at least 1")
|
||||
if args.pimsim_time_ms <= 0:
|
||||
parser.error("--pimsim-time-ms must be positive")
|
||||
raise ValueError("--pimsim-time-ms must be positive")
|
||||
if args.batch_size <= 0:
|
||||
parser.error("--batch-size must be positive")
|
||||
raise ValueError("--batch-size must be positive")
|
||||
if args.timeout_seconds < 0:
|
||||
parser.error("--timeout-seconds must be non-negative")
|
||||
raise ValueError("--timeout-seconds must be non-negative")
|
||||
comparisons_by_arch: dict[str, tuple[tuple[str, int | str, str], ...]] = {}
|
||||
configs_by_arch: dict[str, dict[str, Path]] = {}
|
||||
for arch in args.archs:
|
||||
@@ -697,7 +647,7 @@ def main() -> int:
|
||||
and (args.pipeline is None or comparison[1] == args.pipeline)
|
||||
)
|
||||
if not comparisons:
|
||||
parser.error("--pipeline does not belong to the selected --mode")
|
||||
raise ValueError("--pipeline does not belong to the selected --mode")
|
||||
configs = {
|
||||
mode: config_path(arch, mode, args.pimsim_time_ms, write=not args.dry_run)
|
||||
for mode, _, _ in comparisons
|
||||
@@ -716,7 +666,7 @@ def main() -> int:
|
||||
if not MODELS[name].exists()
|
||||
]
|
||||
if missing:
|
||||
parser.error(f"missing model(s): {', '.join(missing)}")
|
||||
raise ValueError(f"missing model(s): {', '.join(missing)}")
|
||||
|
||||
if args.only is not None:
|
||||
missing_reuse = []
|
||||
@@ -730,7 +680,7 @@ def main() -> int:
|
||||
if not required.exists():
|
||||
missing_reuse.append(str(required))
|
||||
if missing_reuse:
|
||||
parser.error(
|
||||
raise ValueError(
|
||||
f"--only {args.only} needs existing comparison artifacts: "
|
||||
+ ", ".join(missing_reuse)
|
||||
)
|
||||
@@ -740,41 +690,6 @@ def main() -> int:
|
||||
if out_dir is not None and not args.dry_run:
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
print(Style.BRIGHT + f"Found {len(args.models)} Pimcomp model(s) to compare." + Style.RESET_ALL)
|
||||
print(f"Architectures: {', '.join(args.archs)}")
|
||||
print(f"Modes: {', '.join(args.mode)}")
|
||||
print(f"Throughput Pimsim time: {args.pimsim_time_ms} ms")
|
||||
print(f"Max parallel jobs: {args.jobs}")
|
||||
print(f"Shared reference batch: {reference_batch_size}")
|
||||
print(
|
||||
f"Comparison jobs: "
|
||||
f"{sum(len(args.models) * len(comparisons) for comparisons in comparisons_by_arch.values())}"
|
||||
)
|
||||
print(f"Artifacts root: {out_dir or SUITE}")
|
||||
print("=" * 72)
|
||||
|
||||
print_stage("Prepare shared artifacts", STAGE_COLORS["Build runner"])
|
||||
try:
|
||||
for name in args.models:
|
||||
try:
|
||||
run(
|
||||
prepare_common_command(
|
||||
MODELS[name],
|
||||
common_dir(out_dir, name, common_root),
|
||||
args.timeout_seconds,
|
||||
reference_batch_size,
|
||||
args.jobs,
|
||||
),
|
||||
dry_run=args.dry_run,
|
||||
)
|
||||
finally:
|
||||
print_completed(f"shared artifacts: {name}")
|
||||
except KeyboardInterrupt:
|
||||
remove_lock_files(out_dir or SUITE)
|
||||
print("Interrupted; cleaned validation lock files.", file=sys.stderr)
|
||||
return 130
|
||||
|
||||
failed = []
|
||||
comparison_specs: list[ComparisonSpec] = []
|
||||
shared_pimcomp_by_group: dict[tuple[str, str, str], Path] = {}
|
||||
for arch in args.archs:
|
||||
@@ -808,117 +723,171 @@ def main() -> int:
|
||||
)
|
||||
)
|
||||
|
||||
if args.dry_run:
|
||||
for spec in comparison_specs:
|
||||
command = comparison_command_for(
|
||||
spec,
|
||||
args,
|
||||
reuse_shared_pimcomp=not spec.anchor,
|
||||
)
|
||||
if run(command, dry_run=True, check=False):
|
||||
return SuitePlan(args, out_dir, common_root, comparisons_by_arch,
|
||||
reference_batch_size, comparison_specs, {}, {})
|
||||
|
||||
|
||||
def execution_key(plan: SuitePlan, spec: ComparisonSpec) -> tuple:
|
||||
args = plan.args
|
||||
return (spec.model, spec.common_dir, spec.config, spec.mode, spec.pipeline,
|
||||
tuple(args.raptor_extra_args), args.batch_size, args.no_fast,
|
||||
args.threads_per_job)
|
||||
|
||||
|
||||
def build_tasks(plans: list[SuitePlan]) -> list[ExperimentTask]:
|
||||
tasks = []
|
||||
references = {}
|
||||
comparison_sources = {execution_key(plan, spec): spec.output_dir
|
||||
for plan in plans if not plan.args.ablation_variant and plan.args.only is None
|
||||
and not plan.args.raptor_only for spec in plan.specs}
|
||||
for plan in plans:
|
||||
for name in plan.args.models:
|
||||
common = common_dir(plan.root, name, plan.common_root)
|
||||
previous = references.get(common)
|
||||
if previous is None or previous[0] < plan.reference_batch_size:
|
||||
references[common] = (plan.reference_batch_size, name, plan.args)
|
||||
for common, (batch_size, name, args) in references.items():
|
||||
command = prepare_common_command(MODELS[name], common, args.timeout_seconds,
|
||||
batch_size, args.threads_per_job)
|
||||
command += ["--threads-per-job", str(args.threads_per_job)]
|
||||
tasks.append(ExperimentTask("reference:" + str(common), lambda command=command: command,
|
||||
(), common / "prepare.log"))
|
||||
for plan in plans:
|
||||
args = plan.args
|
||||
for spec in plan.specs:
|
||||
source = comparison_sources.get(execution_key(plan, spec)) if args.ablation_variant == "baseline" else None
|
||||
report_root = source if source is not None else spec.output_dir
|
||||
plan.previous_reports[spec.output_dir] = pimcomp_report_stamp(report_root / "pimcomp/output")
|
||||
if args.ablation_variant == "baseline" and execution_key(plan, spec) in comparison_sources:
|
||||
plan.aliases[spec.output_dir] = comparison_sources[execution_key(plan, spec)]
|
||||
continue
|
||||
reuse = not (spec.anchor or args.only == "raptor" or args.raptor_only)
|
||||
dependencies = ["reference:" + str(spec.common_dir)]
|
||||
previous_report = pimcomp_report_stamp(spec.shared_pimcomp_dir) if reuse else None
|
||||
if reuse:
|
||||
dependencies.append(str(spec.shared_pimcomp_dir.parent.parent))
|
||||
|
||||
def command(spec=spec, args=args, reuse=reuse, previous=previous_report):
|
||||
if reuse and not args.dry_run and not pimcomp_report_ready(spec.shared_pimcomp_dir, previous):
|
||||
raise ValueError(f"Pimcomp prerequisite produced no fresh report: {spec.shared_pimcomp_dir}")
|
||||
return comparison_command_for(spec, args, reuse_shared_pimcomp=reuse)
|
||||
|
||||
tasks.append(ExperimentTask(str(spec.output_dir), command, tuple(dependencies),
|
||||
spec.output_dir / "comparison.log"))
|
||||
return tasks
|
||||
|
||||
|
||||
def report_for(plan: SuitePlan, spec: ComparisonSpec) -> Path:
|
||||
return plan.aliases.get(spec.output_dir, spec.output_dir) / "pimcomp/comparison_report.json"
|
||||
|
||||
|
||||
def finish_plan(plan: SuitePlan, results: dict[str, TaskResult]) -> int:
|
||||
args = plan.args
|
||||
failed, diagnostic = [], []
|
||||
overrides = {}
|
||||
for spec in plan.specs:
|
||||
directory = plan.aliases.get(spec.output_dir, spec.output_dir)
|
||||
result = results.get(str(directory))
|
||||
fresh = pimcomp_report_ready(directory / "pimcomp/output", plan.previous_reports[spec.output_dir])
|
||||
overrides[spec.output_dir] = report_for(plan, spec) if result and not result.returncode and fresh else None
|
||||
for arch in args.archs:
|
||||
output = write_results_csv(plan.root, arch, args.models, plan.comparisons[arch],
|
||||
args.ablation_variant, overrides, args.raptor_only)
|
||||
if plan.aliases:
|
||||
destination = (plan.root or SUITE) / "ablation" / args.ablation_variant / "source_reports.json"
|
||||
destination.write_text(json.dumps({str(k): str(v) if v is not None else None
|
||||
for k, v in overrides.items()}, indent=2) + "\n")
|
||||
for spec in plan.specs:
|
||||
result = results.get(str(plan.aliases.get(spec.output_dir, spec.output_dir)))
|
||||
path = report_for(plan, spec)
|
||||
report = json.loads(path.read_text()) if overrides[spec.output_dir] is not None else {}
|
||||
if result is None or result.returncode or overrides[spec.output_dir] is None:
|
||||
failed.append(spec.label)
|
||||
elif not comparison_passed(report):
|
||||
if args.ablation_variant == "no-sync" and completed_synchronization_ablation(report):
|
||||
diagnostic.append(spec.label)
|
||||
else:
|
||||
failed.append(spec.label)
|
||||
elif comparison_specs:
|
||||
raptor_only_run = args.only == "raptor" or args.raptor_only
|
||||
anchor_specs = comparison_specs if raptor_only_run else [
|
||||
spec for spec in comparison_specs if spec.anchor
|
||||
]
|
||||
anchor_jobs = [
|
||||
(
|
||||
spec.label,
|
||||
comparison_command_for(spec, args, reuse_shared_pimcomp=False),
|
||||
None,
|
||||
)
|
||||
for spec in anchor_specs
|
||||
]
|
||||
dependent_specs = [] if raptor_only_run else [
|
||||
spec for spec in comparison_specs if not spec.anchor
|
||||
]
|
||||
print_directly = min(args.jobs, len(comparison_specs)) == 1
|
||||
try:
|
||||
with (nullcontext(None) if print_directly else TemporaryDirectory(prefix="raptor-pimcomp-")) as log_dir:
|
||||
previous_reports = {
|
||||
spec.shared_pimcomp_dir: pimcomp_report_stamp(spec.shared_pimcomp_dir)
|
||||
for spec in anchor_specs
|
||||
}
|
||||
anchor_failed, log_offset = run_comparison_jobs(
|
||||
anchor_jobs,
|
||||
args.jobs,
|
||||
Path(log_dir) if log_dir else None,
|
||||
0,
|
||||
)
|
||||
failed.extend(anchor_failed)
|
||||
dependent_jobs = [
|
||||
(
|
||||
spec.label,
|
||||
comparison_command_for(
|
||||
spec,
|
||||
args,
|
||||
reuse_shared_pimcomp=pimcomp_report_ready(
|
||||
spec.shared_pimcomp_dir, previous_reports[spec.shared_pimcomp_dir]
|
||||
),
|
||||
),
|
||||
None,
|
||||
)
|
||||
for spec in dependent_specs
|
||||
]
|
||||
dependent_failed, _ = run_comparison_jobs(
|
||||
dependent_jobs,
|
||||
args.jobs,
|
||||
Path(log_dir) if log_dir else None,
|
||||
log_offset,
|
||||
)
|
||||
failed.extend(dependent_failed)
|
||||
except KeyboardInterrupt:
|
||||
remove_lock_files(out_dir or SUITE)
|
||||
print("Interrupted; cleaned validation lock files.", file=sys.stderr)
|
||||
return 130
|
||||
|
||||
if args.dry_run:
|
||||
return 1 if failed else 0
|
||||
|
||||
remove_lock_files(out_dir or SUITE)
|
||||
|
||||
results_path = None
|
||||
for arch in args.archs:
|
||||
results_path = write_results_csv(
|
||||
out_dir,
|
||||
arch,
|
||||
args.models,
|
||||
comparisons_by_arch[arch],
|
||||
args.ablation_variant,
|
||||
)
|
||||
assert results_path is not None
|
||||
print_stage(results_path.name, STAGE_COLORS["Compare outputs"])
|
||||
print(results_path.read_text(encoding="utf-8"), end="")
|
||||
diagnostic_failures = []
|
||||
for arch in args.archs:
|
||||
for name in args.models:
|
||||
for mode, pipeline, _ in comparisons_by_arch[arch]:
|
||||
label = f"{arch}/{name}/{mode}/pipeline{pipeline}"
|
||||
report_path = result_dir(
|
||||
out_dir, name, arch, mode, pipeline, args.ablation_variant
|
||||
) / "pimcomp/comparison_report.json"
|
||||
report = json.loads(report_path.read_text(encoding="utf-8")) if report_path.exists() else {}
|
||||
if not comparison_passed(report):
|
||||
if (label not in failed and args.ablation_variant == "no-sync"
|
||||
and completed_synchronization_ablation(report)):
|
||||
diagnostic_failures.append(label)
|
||||
elif label not in failed:
|
||||
failed.append(label)
|
||||
print(Style.BRIGHT + Fore.CYAN + "Summary" + Style.RESET_ALL)
|
||||
total_jobs = sum(len(args.models) * len(comparisons) for comparisons in comparisons_by_arch.values())
|
||||
print(Style.BRIGHT + f"Passed: {total_jobs - len(failed) - len(diagnostic_failures)}" + Style.RESET_ALL)
|
||||
print(Style.BRIGHT + f"Failed: {len(failed)}" + Style.RESET_ALL)
|
||||
if diagnostic_failures:
|
||||
print("No-sync numerical failures (diagnostic only): " + ", ".join(diagnostic_failures))
|
||||
print(Style.BRIGHT + f"Results: {results_path}" + Style.RESET_ALL)
|
||||
print(f"Results: {output}; passed: {len(plan.specs) - len(failed) - len(diagnostic)}; "
|
||||
f"failed: {len(failed)}; diagnostic no-sync failures: {len(diagnostic)}", flush=True)
|
||||
if failed:
|
||||
print(
|
||||
Fore.RED + f"Failed comparisons: {', '.join(failed)}" + Style.RESET_ALL,
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
return 0
|
||||
print("Failed comparisons: " + ", ".join(failed), file=sys.stderr)
|
||||
remove_lock_files(plan.root or SUITE)
|
||||
return int(bool(failed))
|
||||
|
||||
|
||||
def run_plans(plans: list[SuitePlan]) -> int:
|
||||
tasks = build_tasks(plans)
|
||||
args = plans[0].args
|
||||
if any((plan.args.jobs, plan.args.threads_per_job) != (args.jobs, args.threads_per_job) for plan in plans):
|
||||
raise ValueError("all experiment plans must use the same job and thread limits")
|
||||
if args.dry_run:
|
||||
for task in tasks:
|
||||
run(task.command(), dry_run=True, check=False)
|
||||
return 0
|
||||
print(f"Concurrent jobs: {args.jobs}; threads per job: {args.threads_per_job}", flush=True)
|
||||
results = run_tasks(tasks, args.jobs, args.threads_per_job, REPO)
|
||||
# Only the parent writes aggregate CSVs.
|
||||
failures = [finish_plan(plan, results) for plan in plans]
|
||||
return int(any(failures))
|
||||
|
||||
|
||||
def combined_ablation_plans(args: argparse.Namespace) -> tuple[list[SuitePlan], object, Path]:
|
||||
sys.path.insert(0, str(REPO / "validation/tools/pim/ablation"))
|
||||
import run_ablation as ablation
|
||||
from copy import deepcopy
|
||||
root = args.ablation_out_dir or args.out_dir or SUITE
|
||||
models = [name for name in args.models if name in ablation.ABLATION_DEFAULT_MODELS]
|
||||
if not models:
|
||||
raise ValueError("combined ablation needs at least one of the four ablation models")
|
||||
cases = [(mode, depth) for mode, depth, _ in COMPARISONS
|
||||
if mode in args.mode and (depth in (1, 4) if args.pipeline is None else depth == args.pipeline)]
|
||||
if args.pipeline is not None and not cases:
|
||||
cases = [("throughput", args.pipeline)]
|
||||
plans = []
|
||||
for variant in ablation.VARIANTS:
|
||||
for mode, depth in cases:
|
||||
if variant.name == "no-sync" and mode != "throughput":
|
||||
continue
|
||||
selected = deepcopy(args)
|
||||
selected.out_dir, selected.models = root, models
|
||||
selected.common_dir = args.common_dir or args.out_dir or SUITE
|
||||
selected.raptor_only, selected.only = True, None
|
||||
selected.ablation_variant = variant.name
|
||||
selected.mode, selected.pipeline = [mode], depth
|
||||
selected.raptor_extra_args += list(variant.raptor_args)
|
||||
plans.append(make_plan(selected))
|
||||
return plans, ablation, root
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
args = parse_args(argv)
|
||||
if args.clean:
|
||||
print(f"Removed {clean_artifacts(args.out_dir, args.models, args.common_dir)} comparison artifact path(s).")
|
||||
return 0
|
||||
try:
|
||||
plans = [make_plan(args)]
|
||||
if args.with_ablation:
|
||||
if args.only or args.raptor_only or args.ablation_variant:
|
||||
raise ValueError("--with-ablation requires a full compiler comparison")
|
||||
extra, ablation, ablation_root = combined_ablation_plans(args)
|
||||
plans.extend(extra)
|
||||
result = run_plans(plans)
|
||||
if args.with_ablation and not args.dry_run:
|
||||
cases = frozenset((spec.mode, str(spec.pipeline)) for plan in extra for spec in plan.specs)
|
||||
variants = [variant for variant in ablation.VARIANTS
|
||||
if any(plan.args.ablation_variant == variant.name for plan in extra)]
|
||||
_, failures = ablation.aggregate_results(ablation_root, variants, cases,
|
||||
frozenset(extra[0].args.models),
|
||||
output=ablation.DEFAULT_RESULTS_PATH if args.out_dir is None and args.ablation_out_dir is None else None)
|
||||
result = int(bool(result or failures))
|
||||
return result
|
||||
except KeyboardInterrupt:
|
||||
print("Interrupted; terminated experiment workers.", file=sys.stderr)
|
||||
return 130
|
||||
except ValueError as exc:
|
||||
print(str(exc), file=sys.stderr)
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
Reference in New Issue
Block a user