nnicolosi/Raptor
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Some tool drawio and sequence diagram

Browse Source
This commit is contained in:
ilgeco
2026-08-06 14:34:22 +02:00
parent 0712c5ba29
commit 7a3a808ae8
4 changed files with 1246 additions and 69 deletions
Download Patch File Download Diff File Expand all files Collapse all files
validation/tools/pimcomp/compare_raptor_pimcomp.py
+42 -6
View File
@@ -285,6 +285,7 @@ def load_effective_hardware(args: argparse.Namespace) -> dict[str, int]:
def select_pimsim_config(args: argparse.Namespace, hardware: dict[str, int]) -> Path:
fallback: Path | None = None
for path in sorted(PIMSIM_CONFIG_DIR.glob(f"*/{args.pimsim_mode}_config.json")):
with open(path, encoding="utf-8") as f:
config = json.load(f)
@@ -297,15 +298,44 @@ def select_pimsim_config(args: argparse.Namespace, hardware: dict[str, int]) ->
and matrix["xbar_size"] == [hardware["crossbar_size"]] * 2
and network["layout"] == [hardware["mesh_rows"], hardware["mesh_cols"]]
and config["sim_config"]["sim_mode"] == (1 if args.pimsim_mode == "latency" else 0)
and config["sim_config"]["sim_time"] == args.pimsim_time_ms
):
return path
if config["sim_config"]["sim_time"] == args.pimsim_time_ms:
return path
fallback = fallback or path
if fallback is not None:
return fallback
raise ValueError(
f"No pre-generated {args.pimsim_mode} pimsim-nn config matches "
f"{hardware} with sim_time={args.pimsim_time_ms}"
f"No pre-generated {args.pimsim_mode} pimsim-nn config matches {hardware}"
)
def prepare_pimsim_config(
args: argparse.Namespace,
hardware: dict[str, int],
out_dir: Path,
) -> Path:
source = select_pimsim_config(args, hardware)
with open(source, encoding="utf-8") as f:
config = json.load(f)
if config["sim_config"]["sim_time"] == args.pimsim_time_ms:
return source
config["sim_config"]["sim_time"] = args.pimsim_time_ms
target = out_dir / "pimsim_config.json"
target.parent.mkdir(parents=True, exist_ok=True)
target.write_text(json.dumps(config, indent=2) + "\n", encoding="utf-8")
network_path = Path(config["chip_config"]["network_config"]["net_config_file_path"])
if not network_path.is_absolute():
network_path = source.parent / network_path
target_network = target.parent / Path(
config["chip_config"]["network_config"]["net_config_file_path"]
)
target_network.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(network_path, target_network)
return target
def compile_reference(
args: argparse.Namespace,
model_path: Path,
@@ -1221,6 +1251,8 @@ def main():
help="Return a non-zero status if a stage or semantic validation fails.",
)
args = parser.parse_args()
if args.pimsim_time_ms <= 0:
parser.error("--pimsim-time-ms must be positive")
if args.pimcomp_pipeline is None:
args.pimcomp_pipeline = "element" if args.pimsim_mode == "latency" else "batch"
@@ -1485,10 +1517,11 @@ def main():
if not args.skip_pimsim_nn and hardware["core_count"] > 0:
written_config = try_stage(
failures,
"Select pimsim-nn config",
select_pimsim_config,
"Prepare pimsim-nn config",
prepare_pimsim_config,
args,
hardware,
out_dir,
)
if written_config is not None:
pimsim_config = written_config
@@ -1593,8 +1626,11 @@ def main():
"model": str(model_path),
"hardware": hardware,
"pimsim_mode": args.pimsim_mode,
"pimsim_time_ms": args.pimsim_time_ms,
"pimcomp_pipeline": args.pimcomp_pipeline,
"pimcomp_replication": args.pimcomp_replication,
"pimcomp_config": str(args.pimcomp_config),
"raptor_extra_args": args.raptor_extra_arg,
"reused_raptor_report": optional_path(args.reuse_raptor_report.resolve()) if reuse_raptor else None,
"failures": failures,
"steps": [asdict(step) for step in steps],
validation/tools/pimcomp/run_pimcomp_paper_latency.py
+199 -63
View File
@@ -20,57 +20,87 @@ from raptor_validation.pimsim_nn import parse_pimsim_nn_metrics # noqa: E402
from raptor_validation.validate_one import STAGE_COLORS # noqa: E402
PIMCOMP_SOURCE = REPO / "third_party/PIMCOMP-NN"
PIMCOMP_CONFIG = REPO / "validation/pimsim_configs/pimcomp/arch-a/latency_config.json"
PIMCOMP_CONFIGS = REPO / "validation/pimsim_configs/pimcomp"
COMPARE = Path(__file__).resolve().with_name("compare_raptor_pimcomp.py")
ARCHES = tuple(sorted(path.name for path in PIMCOMP_CONFIGS.iterdir() if path.is_dir()))
MODELS = {
"vgg8": SUITE / "vgg8/vgg8-mnist-reconstructed.onnx",
"resnet18": SUITE / "resnet18/resnet18-v1-7.onnx",
"resnet34": SUITE / "resnet34/resnet34-v1-7.onnx",
"googlenet": SUITE / "googlenet/googlenet-12-latency.onnx",
}
COMPARISONS = (
("latency", 1, "element"),
("throughput", 2, "batch"),
("throughput", 4, "batch"),
("throughput", 8, "batch"),
)
def result_dir(root: Path | None, name: str) -> Path:
return root / name if root is not None else MODELS[name].parent
def result_dir(root: Path | None, name: str, mode: str, pipeline: int) -> Path:
base = root / name if root is not None else MODELS[name].parent
suffix = "latency" if mode == "latency" else f"throughput/pipeline{pipeline}"
return base / suffix
def write_results_csv(root: Path | None) -> Path:
def write_results_csv(root: Path | None, arch: str, models: list[str]) -> Path:
output = (root or SUITE) / "results.csv"
fields = (
"model",
"arch",
"mode",
"raptor_pipeline",
"pimcomp_pipeline",
"status",
"raptor_throughput_samples_s",
"pimcomp_throughput_samples_s",
"raptor_latency_ms",
"pimcomp_latency_ms",
"raptor_power_mw",
"pimcomp_power_mw",
"raptor_energy_pj",
"pimcomp_energy_pj",
"faster_compiler",
"better_compiler",
"speedup",
)
rows = []
for name in MODELS:
report_path = result_dir(root, name) / "pimcomp/comparison_report.json"
if not report_path.exists():
continue
report = json.loads(report_path.read_text(encoding="utf-8"))
raptor = report.get("raptor_performance") or {}
pimcomp = report.get("pimcomp_performance") or {}
raptor_latency = raptor.get("latency_ms")
pimcomp_latency = pimcomp.get("latency_ms")
if raptor_latency is None or pimcomp_latency is None:
continue
raptor_energy = (raptor.get("average_energy_pj")
or parse_pimsim_nn_metrics(raptor.get("raw_output", "")).get("average_energy_pj"))
pimcomp_energy = (pimcomp.get("average_energy_pj")
or parse_pimsim_nn_metrics(pimcomp.get("raw_output", "")).get("average_energy_pj"))
faster = "raptor" if raptor_latency < pimcomp_latency else "pimcomp"
rows.append({
"model": name,
"raptor_latency_ms": f"{raptor_latency:.6f}",
"pimcomp_latency_ms": f"{pimcomp_latency:.6f}",
"raptor_energy_pj": "" if raptor_energy is None else f"{raptor_energy:.6f}",
"pimcomp_energy_pj": "" if pimcomp_energy is None else f"{pimcomp_energy:.6f}",
"faster_compiler": faster,
"speedup": f"{max(raptor_latency, pimcomp_latency) / min(raptor_latency, pimcomp_latency):.2f}",
})
for name in models:
for mode, pipeline, pimcomp_pipeline in COMPARISONS:
report_path = result_dir(root, name, mode, pipeline) / "pimcomp/comparison_report.json"
if not report_path.exists():
continue
report = json.loads(report_path.read_text(encoding="utf-8"))
raptor = report.get("raptor_performance") or {}
pimcomp = report.get("pimcomp_performance") or {}
raptor_values = performance_values(raptor)
pimcomp_values = performance_values(pimcomp)
raptor_metric = raptor_values["throughput"] if mode == "throughput" else raptor_values["latency"]
pimcomp_metric = pimcomp_values["throughput"] if mode == "throughput" else pimcomp_values["latency"]
status = "PASS" if comparison_passed(report) else "FAIL"
if raptor_metric is None or pimcomp_metric is None:
better = ""
speedup = ""
else:
better = comparison_winner(mode, raptor_metric, pimcomp_metric)
speedup = f"{max(raptor_metric, pimcomp_metric) / min(raptor_metric, pimcomp_metric):.2f}"
rows.append({
"model": name,
"arch": arch,
"mode": mode,
"raptor_pipeline": pipeline,
"pimcomp_pipeline": pimcomp_pipeline,
"status": status,
"raptor_throughput_samples_s": format_value(raptor_values["throughput"]),
"pimcomp_throughput_samples_s": format_value(pimcomp_values["throughput"]),
"raptor_latency_ms": format_value(raptor_values["latency"]),
"pimcomp_latency_ms": format_value(pimcomp_values["latency"]),
"raptor_power_mw": format_value(raptor_values["power"]),
"pimcomp_power_mw": format_value(pimcomp_values["power"]),
"raptor_energy_pj": format_value(raptor_values["energy"]),
"pimcomp_energy_pj": format_value(pimcomp_values["energy"]),
"better_compiler": better,
"speedup": speedup,
})
with open(output, "w", newline="", encoding="utf-8") as f:
writer = csv.DictWriter(f, fieldnames=fields, lineterminator="\n")
writer.writeheader()
@@ -78,6 +108,47 @@ def write_results_csv(root: Path | None) -> Path:
return output
def performance_values(performance: dict) -> dict[str, float | None]:
parsed = parse_pimsim_nn_metrics(performance.get("raw_output", ""))
return {
"throughput": performance.get("throughput") or parsed.get("throughput"),
"latency": (
performance.get("latency_ms")
or performance.get("average_latency_ms")
or parsed.get("latency_ms")
or parsed.get("average_latency_ms")
),
"power": performance.get("average_power_mw") or parsed.get("average_power_mw"),
"energy": performance.get("average_energy_pj") or parsed.get("average_energy_pj"),
}
def comparison_passed(report: dict) -> bool:
if report.get("failures"):
return False
for key in ("raptor_validation", "pimcomp_validation"):
result = report.get(key) or {}
if result.get("status") != "done" or not result.get("passed"):
return False
for key in ("raptor_performance", "pimcomp_performance"):
performance = report.get(key) or {}
if performance.get("error") or performance.get("skipped"):
return False
return True
def comparison_winner(mode: str, raptor: float, pimcomp: float) -> str:
if raptor == pimcomp:
return "tie"
if mode == "throughput":
return "raptor" if raptor > pimcomp else "pimcomp"
return "raptor" if raptor < pimcomp else "pimcomp"
def format_value(value: float | None) -> str:
return "" if value is None else f"{value:.6f}"
def print_stage(title: str, color: str) -> None:
print("\n" + Style.BRIGHT + color + f"[{title}]" + Style.RESET_ALL, flush=True)
@@ -98,7 +169,17 @@ def validate_pimcomp_source() -> None:
raise RuntimeError(f"PIMCOMP paper setting is missing: {setting}")
def comparison_command(model: Path, result_dir: Path, timeout: float) -> list[str]:
def comparison_command(
model: Path,
result_dir: Path,
config: Path,
mode: str,
pipeline: int,
pimcomp_pipeline: str,
pimsim_time_ms: int,
timeout: float,
) -> list[str]:
time_args = ["--pimsim-time-ms", str(pimsim_time_ms)] if mode == "throughput" else []
return [
sys.executable,
str(COMPARE),
@@ -109,32 +190,49 @@ def comparison_command(model: Path, result_dir: Path, timeout: float) -> list[st
"--pimcomp-dir",
str(PIMCOMP_SOURCE),
"--pimcomp-config",
str(PIMCOMP_CONFIG),
"--core-count",
"168",
"--crossbar-count",
"96",
"--crossbar-size",
"128",
"--mesh-rows",
"12",
"--mesh-cols",
"14",
str(config),
"--pimsim-mode",
"latency",
mode,
*time_args,
"--pimcomp-pipeline",
"element",
pimcomp_pipeline,
"--pimcomp-replication",
"GA",
f"--raptor-extra-arg=--pipeline={pipeline}",
"--timeout-seconds",
str(timeout),
"--fail-on-error",
]
def config_path(arch: str, mode: str) -> Path:
path = PIMCOMP_CONFIGS / arch / f"{mode}_config.json"
if not path.exists():
raise ValueError(f"{arch} has no {mode} config: {path}")
return path
def core_count(config: Path) -> int:
with open(config, encoding="utf-8") as f:
return int(json.load(f)["chip_config"]["core_cnt"])
def completed_report(path: Path, mode: str, pipeline: int, config: Path, pimsim_time_ms: int) -> bool:
if not path.exists():
return False
report = json.loads(path.read_text(encoding="utf-8"))
return (
report.get("pimsim_mode") == mode
and report.get("pimcomp_pipeline") == ("element" if mode == "latency" else "batch")
and report.get("pimsim_time_ms") == pimsim_time_ms
and report.get("pimcomp_config") == str(config.resolve())
and f"--pipeline={pipeline}" in report.get("raptor_extra_args", [])
)
def main() -> int:
parser = argparse.ArgumentParser(
description="Reproduce the serial Arch-A latency comparison from the PIMCOMP paper."
description="Compare supported PIMCOMP models with Raptor latency and throughput schedules."
)
parser.add_argument(
"--out-dir",
@@ -142,6 +240,15 @@ def main() -> int:
help="Result root (default: artifacts beside each model under validation/).",
)
parser.add_argument("--models", nargs="+", choices=MODELS, default=list(MODELS))
parser.add_argument(
"--arch", choices=ARCHES, default="arch-a", help="PIM architecture (default: arch-a)."
)
parser.add_argument(
"--pimsim-time-ms",
type=int,
default=100,
help="throughput pimsim-nn horizon in ms (default: 100).",
)
parser.add_argument("--timeout-seconds", type=float, default=3600.0)
parser.add_argument(
"--resume",
@@ -151,6 +258,16 @@ def main() -> int:
parser.add_argument("--dry-run", action="store_true", help="Print commands without modifying files.")
args = parser.parse_args()
if args.pimsim_time_ms <= 0:
parser.error("--pimsim-time-ms must be positive")
configs = {mode: config_path(args.arch, mode) for mode, _, _ in COMPARISONS}
unsupported = [pipeline for mode, pipeline, _ in COMPARISONS if core_count(configs[mode]) % pipeline]
if unsupported:
parser.error(
f"{args.arch} has {core_count(configs['throughput'])} cores; "
f"throughput pipelines must divide that count (invalid: {unsupported})"
)
out_dir = args.out_dir.resolve() if args.out_dir is not None else None
missing = [str(MODELS[name]) for name in args.models if not MODELS[name].exists()]
@@ -162,6 +279,8 @@ def main() -> int:
out_dir.mkdir(parents=True, exist_ok=True)
print(Style.BRIGHT + f"Found {len(args.models)} PIMCOMP model(s) to compare." + Style.RESET_ALL)
print(f"Architecture: {args.arch}")
print(f"Throughput pimsim time: {args.pimsim_time_ms} ms")
print(f"Results root: {out_dir or SUITE}")
print("=" * 72)
@@ -176,34 +295,51 @@ def main() -> int:
failed = []
for index, name in enumerate(args.models, start=1):
model_result_dir = result_dir(out_dir, name)
print(
"\n" + Fore.CYAN + f"[{index}/{len(args.models)}]" + Style.RESET_ALL
+ f" {Style.BRIGHT}Comparing {name}{Style.RESET_ALL}",
flush=True,
)
if args.resume and (model_result_dir / "pimcomp/comparison_report.json").exists():
for mode, pipeline, pimcomp_pipeline in COMPARISONS:
model_result_dir = result_dir(out_dir, name, mode, pipeline)
print(
Fore.YELLOW + " Completed report exists; skipping" + Style.RESET_ALL,
"\n" + Fore.CYAN + f"[{index}/{len(args.models)}]" + Style.RESET_ALL
+ f" {Style.BRIGHT}Comparing {name} ({mode}, pipeline={pipeline}){Style.RESET_ALL}",
flush=True,
)
continue
returncode = run(
comparison_command(MODELS[name], model_result_dir, args.timeout_seconds),
dry_run=args.dry_run,
check=False,
)
if returncode:
failed.append(name)
if args.resume and completed_report(
model_result_dir / "pimcomp/comparison_report.json",
mode,
pipeline,
configs[mode],
args.pimsim_time_ms,
):
print(
Fore.YELLOW + " Completed report exists; skipping" + Style.RESET_ALL,
flush=True,
)
continue
returncode = run(
comparison_command(
MODELS[name],
model_result_dir,
configs[mode],
mode,
pipeline,
pimcomp_pipeline,
args.pimsim_time_ms,
args.timeout_seconds,
),
dry_run=args.dry_run,
check=False,
)
if returncode:
failed.append(f"{name}/{mode}/pipeline{pipeline}")
if args.dry_run:
return 1 if failed else 0
results_path = write_results_csv(out_dir)
results_path = write_results_csv(out_dir, args.arch, args.models)
print_stage("Results", STAGE_COLORS["Compare Outputs"])
print(results_path.read_text(encoding="utf-8"), end="")
print("\n" + Style.BRIGHT + Fore.CYAN + "Summary" + Style.RESET_ALL)
print(Style.BRIGHT + f"Passed: {len(args.models) - len(failed)}" + Style.RESET_ALL)
total_jobs = len(args.models) * len(COMPARISONS)
print(Style.BRIGHT + f"Passed: {total_jobs - len(failed)}" + Style.RESET_ALL)
print(Style.BRIGHT + f"Failed: {len(failed)}" + Style.RESET_ALL)
print(Style.BRIGHT + f"Results: {results_path}" + Style.RESET_ALL)
if failed: