diff --git a/validation/README.md b/validation/README.md index 2cc8079..abff2a7 100644 --- a/validation/README.md +++ b/validation/README.md @@ -152,7 +152,17 @@ The checked-in profiles are under `validation/pimsim_configs/pimcomp//latency_config.json`. Use `--skip-non-functional-simulation` when latency and power are not required. -The summary reports non-functional results as measured, failed, or skipped. +The summary reports non-functional results as measured, failed, unsupported, or +skipped. + +Overall PASS/FAIL is determined by compilation and functional output +comparison. A non-functional simulation failure remains visible as `ERROR` in +the latency and power columns but does not change a functional PASS. + +`pimsim-nn` does not currently implement the `vsoftmax` instruction. When its +explicit unsupported-op diagnostic is encountered, Softmax validations retain +their functional PASS and show `UNSUPPORTED` in both non-functional columns. +Other `pimsim-nn` failures remain `ERROR`. ## Generated artifacts diff --git a/validation/raptor_validation/validate_one.py b/validation/raptor_validation/validate_one.py index def1668..dbb68b7 100644 --- a/validation/raptor_validation/validate_one.py +++ b/validation/raptor_validation/validate_one.py @@ -2,6 +2,7 @@ import json import os import re import shutil +import subprocess import sys import numpy as np from dataclasses import dataclass, field @@ -47,12 +48,18 @@ MODE_STAGE_TITLES = { PIMSIM_DONE = "DONE" PIMSIM_FAILED = "ERROR" +PIMSIM_UNSUPPORTED = "UNSUPPORTED" PIMSIM_SKIPPED = "SKIP" PIMSIM_NOT_RUN = "-" +PIMSIM_UNSUPPORTED_VSOFTMAX = "pimsim-nn does not support binary opcode vsoftmax" PIMSIM_LATENCY_RE = re.compile(r"\blatency:\s+([0-9.eE+-]+)\s+ms") PIMSIM_POWER_RE = re.compile(r"\baverage power:\s+([0-9.eE+-]+)\s+mW") +class PimSimUnsupportedError(RuntimeError): + pass + + def sanitize_output_name(name): return "".join(ch if ch.isalnum() or ch in "_.-" else "_" for ch in name[:255]) @@ -245,13 +252,19 @@ def pimcomp_compatibility_errors(config_path, *, core_count, crossbar_count, cro def run_pimsim_nn(pimsim_nn_build_dir, pim_dir, config_path, reporter=None, timeout_sec=None): - output = run_command( - [pimsim_nn_build_dir / "ChipTest", pim_dir, config_path, "false"], - cwd=pimsim_nn_build_dir, - reporter=reporter, - timeout_sec=timeout_sec, - capture_output=True, - ) + try: + output = run_command( + [pimsim_nn_build_dir / "ChipTest", pim_dir, config_path, "false"], + cwd=pimsim_nn_build_dir, + reporter=reporter, + timeout_sec=timeout_sec, + capture_output=True, + ) + except subprocess.CalledProcessError as exc: + error_output = exc.output.decode("utf-8", errors="replace") if isinstance(exc.output, bytes) else str(exc.output) + if PIMSIM_UNSUPPORTED_VSOFTMAX in error_output: + raise PimSimUnsupportedError(PIMSIM_UNSUPPORTED_VSOFTMAX) from exc + raise latency_match = PIMSIM_LATENCY_RE.search(output) power_match = PIMSIM_POWER_RE.search(output) if not latency_match or not power_match: @@ -544,6 +557,9 @@ def validate_network(network_onnx_path, raptor_path, onnx_include_dir, reporter, f"Latency: {pimsim_latency_ms:.6f} ms, " f"Power: {pimsim_power_mw:.6f} mW") + except PimSimUnsupportedError as exc: + pimsim_status = PIMSIM_UNSUPPORTED + print_info(reporter, str(exc)) except Exception as exc: pimsim_status = PIMSIM_FAILED reporter.suspend() @@ -559,7 +575,6 @@ def validate_network(network_onnx_path, raptor_path, onnx_include_dir, print_info(reporter, "pimsim-nn non-functional simulation skipped") reporter.advance() - passed = passed and pimsim_status != PIMSIM_FAILED reporter.record_result(passed) status = Fore.GREEN + "PASS" + Style.RESET_ALL if passed else Fore.RED + "FAIL" + Style.RESET_ALL reporter.log(Style.BRIGHT + f"Result: {status}" + Style.RESET_ALL) diff --git a/validation/validate.py b/validation/validate.py index 3442008..9fc00f4 100755 --- a/validation/validate.py +++ b/validation/validate.py @@ -20,6 +20,7 @@ from raptor_validation.validate_one import ( PIMSIM_FAILED, PIMSIM_NOT_RUN, PIMSIM_SKIPPED, + PIMSIM_UNSUPPORTED, ProgressReporter, ValidationResult, clean_workspace_artifacts, @@ -333,7 +334,7 @@ def main(): print(separator) print( f"| {'Operation'.ljust(path_width)} | {'Result'.ljust(status_width)} | " - f"{'Latency'.ljust(latency_width)} | {'Power'.ljust(power_width)} |" + f"{'Latency'.rjust(latency_width)} | {'Power'.rjust(power_width)} |" ) print(separator) for rel, result in results.items(): @@ -342,8 +343,8 @@ def main(): Fore.RED + plain_status.ljust(status_width) + Style.RESET_ALL latency, power = formatted_metrics[rel] print( - f"| {rel.ljust(path_width)} | {status} | {latency.ljust(latency_width)} | " - f"{power.ljust(power_width)} |") + f"| {rel.ljust(path_width)} | {status} | {latency.rjust(latency_width)} | " + f"{power.rjust(power_width)} |") print(separator) print("\n" + Style.BRIGHT + Fore.CYAN + "Summary" + Style.RESET_ALL) print(Style.BRIGHT + f"Passed: {n_passed}" + Style.RESET_ALL) @@ -360,10 +361,14 @@ def main(): result.pimsim_status in (PIMSIM_SKIPPED, PIMSIM_NOT_RUN) for result in results.values() ) + pimsim_unsupported = sum( + result.pimsim_status == PIMSIM_UNSUPPORTED for result in results.values() + ) print( Style.BRIGHT + f"pimsim-nn: {len(measured_latencies)} measured, " - f"{pimsim_failed} failed, {pimsim_skipped} skipped" + f"{pimsim_failed} failed, {pimsim_unsupported} unsupported, " + f"{pimsim_skipped} skipped" + Style.RESET_ALL ) if measured_latencies: