nnicolosi/Raptor
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

better synchronization

Browse Source 
better deadlock detection to also track wait/sync
This commit is contained in:
NiccoloN
2026-08-24 11:58:04 +02:00
parent d634484df2
commit 336f0b506e
20 changed files with 1123 additions and 329 deletions
Download Patch File Download Diff File Expand all files Collapse all files
validation/tools/pim/pimcomp/compare/compare_raptor_pimcomp_model.py
+56 -25
View File
@@ -16,6 +16,7 @@ import sys
import time
import types
from collections import Counter
from concurrent.futures import ThreadPoolExecutor
from dataclasses import asdict, dataclass
from pathlib import Path
from tempfile import TemporaryDirectory
@@ -242,7 +243,7 @@ def reference_inputs_exist(
def reference_batch_dirs(root: Path, batch_size: int) -> list[Path]:
return [root / f"batch_{index:06d}" for index in range(batch_size)]
return [root / f"batch_{index:06d}/outputs" for index in range(batch_size)]
def reference_batch_outputs_exist(
@@ -442,6 +443,7 @@ def generate_reference_batch_outputs(
runner_build_dir: Path,
model_path: Path,
input_batch: list[list[np.ndarray]],
outputs_desc: list[tuple[int, str, int, list[int]]],
steps: list[StepRecord],
args: argparse.Namespace,
out_dir: Path,
@@ -450,20 +452,26 @@ def generate_reference_batch_outputs(
) -> list[Path]:
if print_header:
print_step("Run reference")
references = []
for index, sample in enumerate(input_batch):
references.append(
generate_reference_outputs(
runner_path,
runner_build_dir,
model_path,
sample,
steps,
args,
out_dir / f"batch_{index:06d}",
print_header=False,
)
references = reference_batch_dirs(out_dir, len(input_batch))
missing = [
index for index, reference in enumerate(references)
if not reference_outputs_exist(outputs_desc, reference)
]
def generate(index: int) -> None:
generate_reference_outputs(
runner_path,
runner_build_dir,
model_path,
input_batch[index],
steps,
args,
out_dir / f"batch_{index:06d}",
print_header=False,
)
with ThreadPoolExecutor(max_workers=args.jobs) as executor:
list(executor.map(generate, missing))
return references
@@ -490,6 +498,7 @@ def prepare_reference_batch_outputs(
runner_build_dir,
model_path,
input_batch,
outputs_desc,
steps,
args,
out_dir,
@@ -512,16 +521,29 @@ def prepare_common_artifacts(
outputs_ready = reference_outputs_exist(outputs_desc, outputs_dir)
if inputs_ready:
arrays_in_order = load_saved_inputs(inputs_desc, inputs_dir)
input_batch = generate_input_batch(
inputs_desc, arrays_in_order, args.batch_size, args.seed)
batch_dir = common_dir / f"reference_seed_{args.seed}"
first_batch_dir = batch_dir / "batch_000000"
if inputs_ready and outputs_ready and not reference_outputs_exist(
outputs_desc, first_batch_dir / "outputs"
):
shutil.copytree(inputs_dir, first_batch_dir / "inputs", dirs_exist_ok=True)
shutil.copytree(outputs_dir, first_batch_dir / "outputs", dirs_exist_ok=True)
references = prepare_reference_batch_outputs(
runner_path,
runner_path.parent,
model_path,
input_batch,
outputs_desc,
steps,
args,
batch_dir,
)
if not (inputs_ready and outputs_ready):
generate_reference_outputs(
runner_path,
runner_path.parent,
model_path,
arrays_in_order,
steps,
args,
common_dir,
)
shutil.copytree(
references[0].parent / "inputs", inputs_dir, dirs_exist_ok=True)
shutil.copytree(references[0], outputs_dir, dirs_exist_ok=True)
def compile_raptor_target(
model_path: Path,
out_dir: Path,
@@ -1409,7 +1431,14 @@ def main():
parser.add_argument("--mesh-cols", type=int)
parser.add_argument("--pimsim-time-ms", type=int, default=1000)
parser.add_argument("--pimsim-mode", choices=["latency", "throughput"], default="latency")
parser.add_argument("--batch-size", type=int, default=128)
parser.add_argument("--batch-size", type=int, default=64)
parser.add_argument(
"-j",
"--jobs",
type=int,
default=4,
help="Maximum parallel native reference runner processes (default: 4).",
)
parser.add_argument("--pimcomp-pipeline", choices=["element", "batch"])
parser.add_argument("--pimcomp-model-name", help="Use a Pimcomp built-in model name such as vgg16.")
parser.add_argument(
@@ -1466,6 +1495,8 @@ def main():
parser.error("--pimsim-time-ms must be positive")
if args.batch_size <= 0:
parser.error("--batch-size must be positive")
if args.jobs <= 0:
parser.error("--jobs must be positive")
if args.pimsim_mode == "throughput" and args.batch_size < 2:
parser.error("throughput mode requires batch size greater than 1")
if args.timeout_seconds < 0:
@@ -1674,7 +1705,7 @@ def main():
write_input_batch_csv(out_dir / "inputs.csv", input_batch)
raptor_input_bins = write_input_batch_binaries(input_batch, out_dir / "simulation/raptor_inputs")
if args.pimsim_mode == "throughput":
batch_reference_dir = common_dir / f"reference_batch_{args.batch_size}_seed_{args.seed}"
batch_reference_dir = common_dir / f"reference_seed_{args.seed}"
throughput_references = try_stage(
failures,
"Run reference",
validation/tools/pim/pimcomp/compare/run_pimcomp_models.py
+24 -23
View File
@@ -261,23 +261,9 @@ def performance_values(performance: dict) -> dict[str, float | None]:
}
def comparison_passed(report: dict, compiler: str | None = None) -> bool:
other_compiler = "PIMCOMP" if compiler == "raptor" else "RAPTOR"
if any(
compiler is None or other_compiler not in failure.get("stage", "").upper()
for failure in report.get("failures", [])
):
return False
compilers = (compiler,) if compiler is not None else ("raptor", "pimcomp")
for name in compilers:
result = report.get(f"{name}_validation") or {}
if result.get("status") != "done" or not result.get("passed"):
return False
for name in compilers:
performance = report.get(f"{name}_performance") or {}
if performance.get("error") or performance.get("skipped"):
return False
return True
def comparison_passed(report: dict) -> bool:
result = report.get("raptor_validation") or {}
return result.get("status") == "done" and bool(result.get("passed"))
def functional_validation_status(result: dict | None) -> str:
@@ -404,13 +390,18 @@ def comparison_command(
*[f"--raptor-extra-arg={arg}" for arg in raptor_extra_args],
"--timeout-seconds",
str(timeout),
"--fail-on-error",
*([] if fast else ["--no-fast"]),
*reuse_args,
]
def prepare_common_command(model: Path, common_dir: Path, timeout: float) -> list[str]:
def prepare_common_command(
model: Path,
common_dir: Path,
timeout: float,
batch_size: int,
jobs: int,
) -> list[str]:
return [
sys.executable,
str(COMPARE),
@@ -421,6 +412,10 @@ def prepare_common_command(model: Path, common_dir: Path, timeout: float) -> lis
"--common-dir",
str(common_dir),
"--prepare-common",
"--batch-size",
str(batch_size),
"--jobs",
str(jobs),
"--timeout-seconds",
str(timeout),
]
@@ -587,8 +582,8 @@ def main() -> int:
parser.add_argument(
"--batch-size",
type=int,
default=128,
help="functional throughput batch size (default: 128).",
default=64,
help="functional throughput and shared reference batch size (default: 64).",
)
parser.add_argument(
"--timeout-seconds",
@@ -664,6 +659,11 @@ def main() -> int:
}
comparisons_by_arch[arch] = comparisons
configs_by_arch[arch] = configs
reference_batch_size = max(
1 if mode == "latency" else args.batch_size
for comparisons in comparisons_by_arch.values()
for mode, _, _ in comparisons
)
missing = [
str(path)
@@ -706,6 +706,7 @@ def main() -> int:
print(f"Modes: {', '.join(args.mode)}")
print(f"Throughput Pimsim time: {args.pimsim_time_ms} ms")
print(f"Max parallel jobs: {args.jobs}")
print(f"Shared reference batch: {reference_batch_size}")
print(
f"Comparison jobs: "
f"{sum(len(args.models) * len(comparisons) for comparisons in comparisons_by_arch.values())}"
@@ -722,6 +723,8 @@ def main() -> int:
FUNCTIONAL_MODELS[name],
common_dir(out_dir, name, common_root),
args.timeout_seconds,
reference_batch_size,
args.jobs,
),
dry_run=args.dry_run,
)
@@ -849,10 +852,8 @@ def main() -> int:
report_path = result_dir(
out_dir, name, arch, mode, pipeline, args.ablation_variant
) / "pimcomp/comparison_report.json"
compiler = "raptor" if args.raptor_only else args.only
if not report_path.exists() or not comparison_passed(
json.loads(report_path.read_text(encoding="utf-8")),
compiler,
):
if label not in failed:
failed.append(label)