fix validation for unsupported operation

validation/README.md

@@ -152,7 +152,17 @@ The checked-in profiles are under
 `validation/pimsim_configs/pimcomp/<profile>/latency_config.json`.
 
 Use `--skip-non-functional-simulation` when latency and power are not required.
-The summary reports non-functional results as measured, failed, or skipped.
+The summary reports non-functional results as measured, failed, unsupported, or
+skipped.
+
+Overall PASS/FAIL is determined by compilation and functional output
+comparison. A non-functional simulation failure remains visible as `ERROR` in
+the latency and power columns but does not change a functional PASS.
+
+`pimsim-nn` does not currently implement the `vsoftmax` instruction. When its
+explicit unsupported-op diagnostic is encountered, Softmax validations retain
+their functional PASS and show `UNSUPPORTED` in both non-functional columns.
+Other `pimsim-nn` failures remain `ERROR`.
 
 ## Generated artifacts
 

validation/raptor_validation/validate_one.py

@@ -2,6 +2,7 @@ import json
 import os
 import re
 import shutil
+import subprocess
 import sys
 import numpy as np
 from dataclasses import dataclass, field
@@ -47,12 +48,18 @@ MODE_STAGE_TITLES = {
 
 PIMSIM_DONE = "DONE"
 PIMSIM_FAILED = "ERROR"
+PIMSIM_UNSUPPORTED = "UNSUPPORTED"
 PIMSIM_SKIPPED = "SKIP"
 PIMSIM_NOT_RUN = "-"
+PIMSIM_UNSUPPORTED_VSOFTMAX = "pimsim-nn does not support binary opcode vsoftmax"
 PIMSIM_LATENCY_RE = re.compile(r"\blatency:\s+([0-9.eE+-]+)\s+ms")
 PIMSIM_POWER_RE = re.compile(r"\baverage power:\s+([0-9.eE+-]+)\s+mW")
 
 
+class PimSimUnsupportedError(RuntimeError):
+    pass
+
+
 def sanitize_output_name(name):
     return "".join(ch if ch.isalnum() or ch in "_.-" else "_" for ch in name[:255])
 
@@ -245,13 +252,19 @@ def pimcomp_compatibility_errors(config_path, *, core_count, crossbar_count, cro
 
 
 def run_pimsim_nn(pimsim_nn_build_dir, pim_dir, config_path, reporter=None, timeout_sec=None):
-    output = run_command(
+    try:
-        [pimsim_nn_build_dir / "ChipTest", pim_dir, config_path, "false"],
+        output = run_command(
-        cwd=pimsim_nn_build_dir,
+            [pimsim_nn_build_dir / "ChipTest", pim_dir, config_path, "false"],
-        reporter=reporter,
+            cwd=pimsim_nn_build_dir,
-        timeout_sec=timeout_sec,
+            reporter=reporter,
-        capture_output=True,
+            timeout_sec=timeout_sec,
-    )
+            capture_output=True,
+        )
+    except subprocess.CalledProcessError as exc:
+        error_output = exc.output.decode("utf-8", errors="replace") if isinstance(exc.output, bytes) else str(exc.output)
+        if PIMSIM_UNSUPPORTED_VSOFTMAX in error_output:
+            raise PimSimUnsupportedError(PIMSIM_UNSUPPORTED_VSOFTMAX) from exc
+        raise
     latency_match = PIMSIM_LATENCY_RE.search(output)
     power_match = PIMSIM_POWER_RE.search(output)
     if not latency_match or not power_match:
@@ -544,6 +557,9 @@ def validate_network(network_onnx_path, raptor_path, onnx_include_dir,
                     reporter,
                     f"Latency: {pimsim_latency_ms:.6f} ms, "
                     f"Power: {pimsim_power_mw:.6f} mW")
+            except PimSimUnsupportedError as exc:
+                pimsim_status = PIMSIM_UNSUPPORTED
+                print_info(reporter, str(exc))
             except Exception as exc:
                 pimsim_status = PIMSIM_FAILED
                 reporter.suspend()
@@ -559,7 +575,6 @@ def validate_network(network_onnx_path, raptor_path, onnx_include_dir,
             print_info(reporter, "pimsim-nn non-functional simulation skipped")
         reporter.advance()
 
-        passed = passed and pimsim_status != PIMSIM_FAILED
         reporter.record_result(passed)
         status = Fore.GREEN + "PASS" + Style.RESET_ALL if passed else Fore.RED + "FAIL" + Style.RESET_ALL
         reporter.log(Style.BRIGHT + f"Result: {status}" + Style.RESET_ALL)

validation/validate.py

@@ -20,6 +20,7 @@ from raptor_validation.validate_one import (
     PIMSIM_FAILED,
     PIMSIM_NOT_RUN,
     PIMSIM_SKIPPED,
+    PIMSIM_UNSUPPORTED,
     ProgressReporter,
     ValidationResult,
     clean_workspace_artifacts,
@@ -333,7 +334,7 @@ def main():
     print(separator)
     print(
         f"| {'Operation'.ljust(path_width)} | {'Result'.ljust(status_width)} | "
-        f"{'Latency'.ljust(latency_width)} | {'Power'.ljust(power_width)} |"
+        f"{'Latency'.rjust(latency_width)} | {'Power'.rjust(power_width)} |"
     )
     print(separator)
     for rel, result in results.items():
@@ -342,8 +343,8 @@ def main():
             Fore.RED + plain_status.ljust(status_width) + Style.RESET_ALL
         latency, power = formatted_metrics[rel]
         print(
-            f"| {rel.ljust(path_width)} | {status} | {latency.ljust(latency_width)} | "
+            f"| {rel.ljust(path_width)} | {status} | {latency.rjust(latency_width)} | "
-            f"{power.ljust(power_width)} |")
+            f"{power.rjust(power_width)} |")
     print(separator)
     print("\n" + Style.BRIGHT + Fore.CYAN + "Summary" + Style.RESET_ALL)
     print(Style.BRIGHT + f"Passed: {n_passed}" + Style.RESET_ALL)
@@ -360,10 +361,14 @@ def main():
         result.pimsim_status in (PIMSIM_SKIPPED, PIMSIM_NOT_RUN)
         for result in results.values()
     )
+    pimsim_unsupported = sum(
+        result.pimsim_status == PIMSIM_UNSUPPORTED for result in results.values()
+    )
     print(
         Style.BRIGHT
         + f"pimsim-nn: {len(measured_latencies)} measured, "
-        f"{pimsim_failed} failed, {pimsim_skipped} skipped"
+        f"{pimsim_failed} failed, {pimsim_unsupported} unsupported, "
+        f"{pimsim_skipped} skipped"
         + Style.RESET_ALL
     )
     if measured_latencies: