nnicolosi/Raptor
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

add throughput mode to validation scripts

Browse Source 
make raptor also emit input sizes
This commit is contained in:
NiccoloN
2026-08-11 10:34:50 +02:00
parent 910701dfaf
commit c55d9f3dad
15 changed files with 974 additions and 497 deletions
Download Patch File Download Diff File Expand all files Collapse all files
validation/validate.py
+133 -52
View File
@@ -34,6 +34,8 @@ from raptor_validation.raptor import PIM_PASS_LABELS
DEFAULT_PIMCOMP_CONFIG = "arch-a"
PIMCOMP_CONFIG_CHOICES = ("arch-a", "arch-b", "arch-c")
THROUGHPUT_PIPELINE = 4
THROUGHPUT_BATCH_SIZE = 4
def discover_onnx_files(root):
@@ -67,9 +69,13 @@ def run_validation_job(job):
)
except Exception as exc:
print_validation_error(reporter, rel, exc)
return ValidationResult(False, pimsim_status=PIMSIM_NOT_RUN)
finally:
reporter.finish()
return ValidationResult(
False,
latency_passed=False,
throughput_passed=False,
pimsim_status=PIMSIM_NOT_RUN,
throughput_pimsim_status=PIMSIM_NOT_RUN,
)
sys.stdout.flush()
sys.stderr.flush()
@@ -89,6 +95,7 @@ def run_validation_job(job):
os.dup2(saved_stderr, 2)
os.close(saved_stdout)
os.close(saved_stderr)
reporter.finish()
completed.append((str(rel), result, str(log_path) if log_path else None))
return completed
@@ -135,10 +142,10 @@ def print_average_pim_pass_timings(pass_timing_sums, pass_timing_counts, total_t
print(f" {'Total'.ljust(28)} {total_timing_sum / timed_benchmark_count:.4f}s")
def format_pimsim_metric(result, value, unit):
if result.pimsim_status == PIMSIM_DONE:
return f"{value:.6f} {unit}"
return result.pimsim_status
def format_pimsim_metric(status, value, unit):
if status == PIMSIM_DONE:
return f"{value:.2f} {unit}"
return status
def format_memory(byte_count):
@@ -147,6 +154,47 @@ def format_memory(byte_count):
return f"{byte_count / (1 << 20):.2f} MiB"
def print_results_table(title, headers, rows):
widths = [max(len(header), *(len(row[index]) for row in rows))
for index, header in enumerate(headers)]
separator = "+-" + "-+-".join("-" * width for width in widths) + "-+"
def table_line(row):
return "| " + " | ".join(
value.ljust(widths[index]) if index < 3 else value.rjust(widths[index])
for index, value in enumerate(row)) + " |"
print("\n" + Style.BRIGHT + Fore.CYAN + title + Style.RESET_ALL)
print(separator)
print(table_line(headers))
print(separator)
for row in rows:
line = table_line(row)
status = row[2].ljust(widths[2])
color = Fore.GREEN if row[2] == "PASS" else Fore.RED
print(line.replace(status, color + status + Style.RESET_ALL, 1))
print(separator)
def mode_common_metrics(result, mode):
metrics = result.mode_metrics.get(mode, {})
fallback = {
"compile_time_s": result.compile_time_s if mode == "latency" else None,
"host_memory_bytes": result.host_memory_bytes if mode == "latency" else None,
"cores_memory_bytes": result.cores_memory_bytes if mode == "latency" else None,
"used_core_count": result.used_core_count if mode == "latency" else None,
"used_crossbar_count": result.used_crossbar_count if mode == "latency" else None,
}
values = {**fallback, **metrics}
return (
f"{values['compile_time_s']:.3f} s" if values["compile_time_s"] is not None else "-",
format_memory(values["host_memory_bytes"]),
format_memory(values["cores_memory_bytes"]),
str(values["used_core_count"]) if values["used_core_count"] is not None else "-",
str(values["used_crossbar_count"]) if values["used_crossbar_count"] is not None else "-",
)
def operation_label(relative_path):
path = Path(relative_path)
return str(path.parent) if path.parent != Path(".") else path.stem
@@ -172,6 +220,8 @@ def main():
"(default: arch-a).")
ap.add_argument("--skip-non-functional-simulation", action="store_true",
help="Skip non-functional simulation.")
ap.add_argument("--no-fast", action="store_true",
help="Disable fast pimsim-nn throughput convergence for authoritative experiments.")
ap.add_argument("--threshold", type=float, default=1e-3,
help="Absolute tolerance for per-element output comparison.")
ap.add_argument("--relative-threshold", type=float, default=1e-5,
@@ -187,8 +237,8 @@ def main():
help="Additional argument to pass through to the Raptor compiler. Repeat as needed.")
ap.add_argument("--command-timeout-seconds", type=float, default=1000000.0,
help="Per-subprocess timeout in seconds for compiler, runner, and simulation commands.")
ap.add_argument("-j", "--jobs", type=int, default=os.cpu_count() or 1,
help="Number of model validations to run in parallel (default: all available CPUs).")
ap.add_argument("-j", "--jobs", type=int, default=4,
help="Number of model validations to run in parallel (default: 4).")
ap.add_argument("--clean", action="store_true",
help="Remove generated validation artifacts under each model workspace and exit.")
mode_group = ap.add_mutually_exclusive_group()
@@ -214,6 +264,9 @@ def main():
script_dir / ".." / "backend-simulators" / "pim" / "pimsim-nn" / "build"
)
pimsim_config_path = pimcomp_configs_dir / a.pimcomp_config / "latency_config.json"
throughput_pimsim_config_path = (
pimcomp_configs_dir / a.pimcomp_config / "throughput_config_1000ms.json"
)
if not operations_dir.is_dir():
print(Fore.RED + f"Operations directory not found: {operations_dir}" + Style.RESET_ALL)
@@ -252,6 +305,7 @@ def main():
mode = MODE_RUN_ONLY
selected_pimsim_config = None
selected_throughput_pimsim_config = None
if not a.skip_non_functional_simulation:
compatibility_errors = pimcomp_compatibility_errors(
pimsim_config_path,
@@ -269,6 +323,7 @@ def main():
)
else:
selected_pimsim_config = pimsim_config_path
selected_throughput_pimsim_config = throughput_pimsim_config_path
results = {} # relative_path -> ValidationResult
pass_timing_sums = {label: 0.0 for _, label in PIM_PASS_LABELS}
@@ -290,6 +345,10 @@ def main():
"raptor_extra_args": raptor_extra_args,
"pimsim_nn_build_dir": pimsim_nn_build_dir,
"pimsim_config_path": selected_pimsim_config,
"throughput_pipeline": THROUGHPUT_PIPELINE,
"throughput_batch_size": THROUGHPUT_BATCH_SIZE,
"throughput_pimsim_config_path": selected_throughput_pimsim_config,
"pimsim_fast": not a.no_fast,
"command_timeout_seconds": a.command_timeout_seconds,
"threshold": a.threshold,
"rtol": a.relative_threshold,
@@ -356,44 +415,68 @@ def main():
# Summary
n_passed = sum(1 for result in results.values() if result.passed)
n_total = len(results)
headers = ("Operation", "Result", "Compile", "Host mem", "Cores mem",
"Cores", "Xbars", "Latency", "Power", "Energy")
rows = []
latency_headers = ("Operation", "Arch", "Result", "Compile", "Host mem", "Cores mem",
"Cores", "Xbars", "Latency", "Power", "Energy")
throughput_headers = ("Operation", "Arch", "Result", "Compile", "Host mem", "Cores mem",
"Cores", "Xbars", "Throughput", "Avg latency", "Avg power",
"Avg energy")
csv_headers = (
"Operation", "Arch", "Result (l)", "Result (t)",
"Compile (l)", "Host mem (l)", "Cores mem (l)", "Cores (l)", "Xbars (l)",
"Latency (l)", "Power (l)", "Energy (l)",
"Compile (t)", "Host mem (t)", "Cores mem (t)", "Cores (t)", "Xbars (t)",
"Avg latency (t)", "Throughput (t)", "Avg power (t)", "Avg energy (t)",
)
latency_rows = []
throughput_rows = []
csv_rows = []
for rel, result in results.items():
rows.append((
operation_label(rel), "PASS" if result.passed else "FAIL",
f"{result.compile_time_s:.3f} s" if result.compile_time_s is not None else "-",
format_memory(result.host_memory_bytes),
format_memory(result.cores_memory_bytes),
str(result.used_core_count) if result.used_core_count is not None else "-",
str(result.used_crossbar_count) if result.used_crossbar_count is not None else "-",
format_pimsim_metric(result, result.pimsim_latency_ms, "ms"),
format_pimsim_metric(result, result.pimsim_power_mw, "mW"),
format_pimsim_metric(result, result.pimsim_energy_pj, "pJ"),
operation = operation_label(rel)
latency_status = "PASS" if result.latency_passed else "FAIL"
throughput_status = "PASS" if result.throughput_passed else "FAIL"
latency_common = mode_common_metrics(result, "latency")
throughput_common = mode_common_metrics(result, "throughput")
latency_metrics = (
format_pimsim_metric(result.pimsim_status, result.pimsim_latency_ms, "ms"),
format_pimsim_metric(result.pimsim_status, result.pimsim_power_mw, "mW"),
format_pimsim_metric(result.pimsim_status, result.pimsim_energy_pj, "pJ"),
)
throughput_metrics = (
format_pimsim_metric(
result.throughput_pimsim_status,
result.pimsim_throughput_samples_s,
"samples/s",
),
format_pimsim_metric(
result.throughput_pimsim_status,
result.pimsim_throughput_average_latency_ms,
"ms",
),
format_pimsim_metric(
result.throughput_pimsim_status,
result.pimsim_throughput_average_power_mw,
"mW",
),
format_pimsim_metric(
result.throughput_pimsim_status,
result.pimsim_throughput_average_energy_pj,
"pJ/it",
),
)
latency_rows.append((operation, a.pimcomp_config, latency_status, *latency_common, *latency_metrics))
throughput_rows.append((operation, a.pimcomp_config, throughput_status, *throughput_common, *throughput_metrics))
csv_rows.append((
operation, a.pimcomp_config, latency_status, throughput_status,
*latency_common, *latency_metrics,
*throughput_common, *throughput_metrics,
))
widths = [max(len(header), *(len(row[index]) for row in rows))
for index, header in enumerate(headers)]
separator = "+-" + "-+-".join("-" * width for width in widths) + "-+"
def table_line(row):
return "| " + " | ".join(
value.ljust(widths[index]) if index < 2 else value.rjust(widths[index])
for index, value in enumerate(row)) + " |"
print(separator)
print(table_line(headers))
print(separator)
for row in rows:
line = table_line(row)
color = Fore.GREEN if row[1] == "PASS" else Fore.RED
line = line.replace(row[1].ljust(widths[1]),
color + row[1].ljust(widths[1]) + Style.RESET_ALL, 1)
print(line)
print(separator)
print_results_table("Latency", latency_headers, latency_rows)
print_results_table("Throughput", throughput_headers, throughput_rows)
with (operations_dir / "validation_results.csv").open(
"w", encoding="utf-8", newline=""
) as results_file:
csv.writer(results_file).writerows((headers, *rows))
csv.writer(results_file, lineterminator="\n").writerows((csv_headers, *csv_rows))
print("\n" + Style.BRIGHT + Fore.CYAN + "Summary" + Style.RESET_ALL)
print(Style.BRIGHT + f"Passed: {n_passed}" + Style.RESET_ALL)
print(Style.BRIGHT + f"Failed: {n_total - n_passed}" + Style.RESET_ALL)
@@ -402,19 +485,17 @@ def main():
for result in results.values()
if result.pimsim_status == PIMSIM_DONE
]
pimsim_failed = sum(
result.pimsim_status == PIMSIM_FAILED for result in results.values()
)
pimsim_skipped = sum(
result.pimsim_status in (PIMSIM_SKIPPED, PIMSIM_NOT_RUN)
pimsim_statuses = [
status
for result in results.values()
)
pimsim_unsupported = sum(
result.pimsim_status == PIMSIM_UNSUPPORTED for result in results.values()
)
for status in (result.pimsim_status, result.throughput_pimsim_status)
]
pimsim_failed = pimsim_statuses.count(PIMSIM_FAILED)
pimsim_skipped = sum(status in (PIMSIM_SKIPPED, PIMSIM_NOT_RUN) for status in pimsim_statuses)
pimsim_unsupported = pimsim_statuses.count(PIMSIM_UNSUPPORTED)
print(
Style.BRIGHT
+ f"pimsim-nn: {len(measured_latencies)} measured, "
+ f"pimsim-nn: {pimsim_statuses.count(PIMSIM_DONE)} measured, "
f"{pimsim_failed} failed, {pimsim_unsupported} unsupported, "
f"{pimsim_skipped} skipped"
+ Style.RESET_ALL
@@ -422,7 +503,7 @@ def main():
if measured_latencies:
print(
Style.BRIGHT
+ f"Total latency: {sum(measured_latencies):.6f} ms"
+ f"Total latency: {sum(measured_latencies):.2f} ms"
+ Style.RESET_ALL
)
if a.verbose: