finally fast googlenet with correct latency artifacts for fair comparison

validation/raptor_validation/validate_one.py

@@ -1,6 +1,5 @@
 import json
 import os
-import re
 import shutil
 import subprocess
 import sys
@@ -11,6 +10,7 @@ from colorama import Style, Fore
 from .gen_network_runner import gen_network_runner
 from .onnx_utils import gen_random_inputs, save_inputs_to_files, onnx_io, write_inputs_to_memory_bin, _ONNX_TO_NP
 from .raptor import compile_with_raptor
+from .pimsim_nn import export_raptor_latency_artifact, parse_pimsim_nn_metrics
 from .subprocess_utils import run_command_with_reporter
 
 STAGE_TITLES = (
@@ -61,9 +61,7 @@ PIMSIM_FAILED = "ERROR"
 PIMSIM_UNSUPPORTED = "UNSUPPORTED"
 PIMSIM_SKIPPED = "SKIP"
 PIMSIM_NOT_RUN = "-"
-PIMSIM_UNSUPPORTED_VSOFTMAX = "pimsim-nn does not support binary opcode vsoftmax"
-PIMSIM_LATENCY_RE = re.compile(r"\blatency:\s+([0-9.eE+-]+)\s+ms")
-PIMSIM_POWER_RE = re.compile(r"\baverage power:\s+([0-9.eE+-]+)\s+mW")
+PIMSIM_UNSUPPORTED_VSOFTMAX = "pimsim-nn does not support opcode vsoftmax"
 
 
 class PimSimUnsupportedError(RuntimeError):
@@ -80,6 +78,7 @@ class ValidationResult:
     pim_pass_timings: dict[str, float] = field(default_factory=dict)
     pimsim_latency_ms: float | None = None
     pimsim_power_mw: float | None = None
+    pimsim_energy_pj: float | None = None
     pimsim_status: str = PIMSIM_SKIPPED
 
 
@@ -262,9 +261,10 @@ def pimcomp_compatibility_errors(config_path, *, core_count, crossbar_count, cro
 
 
 def run_pimsim_nn(pimsim_nn_build_dir, pim_dir, config_path, reporter=None, timeout_sec=None):
+    latency_artifact = export_raptor_latency_artifact(pim_dir, Path(pim_dir).parent / "pimsim_nn")
     try:
         output = run_command(
-            [pimsim_nn_build_dir / "ChipTest", pim_dir, config_path, "--gui=false"],
+            [pimsim_nn_build_dir / "ChipTest", latency_artifact, config_path, "--gui=false"],
             cwd=pimsim_nn_build_dir,
             reporter=reporter,
             timeout_sec=timeout_sec,
@@ -275,11 +275,11 @@ def run_pimsim_nn(pimsim_nn_build_dir, pim_dir, config_path, reporter=None, time
         if PIMSIM_UNSUPPORTED_VSOFTMAX in error_output:
             raise PimSimUnsupportedError(PIMSIM_UNSUPPORTED_VSOFTMAX) from exc
         raise
-    latency_match = PIMSIM_LATENCY_RE.search(output)
-    power_match = PIMSIM_POWER_RE.search(output)
-    if not latency_match or not power_match:
-        raise RuntimeError("pimsim-nn output did not contain latency and average power")
-    return float(latency_match.group(1)), float(power_match.group(1))
+    metrics = parse_pimsim_nn_metrics(output)
+    required = ("latency_ms", "average_power_mw", "average_energy_pj")
+    if any(name not in metrics for name in required):
+        raise RuntimeError("pimsim-nn output did not contain latency, average power, and average energy")
+    return tuple(metrics[name] for name in required)
 
 
 def clean_workspace_artifacts(workspace_dir, model_stem):
@@ -416,6 +416,8 @@ def validate_network(network_onnx_path, raptor_path, onnx_include_dir,
         pimsim_nn_build_dir = Path(pimsim_nn_build_dir).resolve()
         pimsim_config_path = Path(pimsim_config_path).resolve()
     compile_extra_args = list(raptor_extra_args or [])
+    if pimsim_enabled and "--pim-emit-json" not in compile_extra_args:
+        compile_extra_args.append("--pim-emit-json")
     owns_reporter = reporter is None
     reporter = reporter or ProgressReporter(model_total, stages_per_model=len(MODE_STAGE_TITLES[mode]), verbose=verbose)
 
@@ -540,10 +542,11 @@ def validate_network(network_onnx_path, raptor_path, onnx_include_dir,
             "Run Non-functional Simulation")
         pimsim_latency_ms = None
         pimsim_power_mw = None
+        pimsim_energy_pj = None
         pimsim_status = PIMSIM_SKIPPED
         if pimsim_enabled:
             try:
-                pimsim_latency_ms, pimsim_power_mw = run_pimsim_nn(
+                pimsim_latency_ms, pimsim_power_mw, pimsim_energy_pj = run_pimsim_nn(
                     pimsim_nn_build_dir,
                     pim_dir,
                     pimsim_config_path,
@@ -554,7 +557,8 @@ def validate_network(network_onnx_path, raptor_path, onnx_include_dir,
                 print_info(
                     reporter,
                     f"Latency: {pimsim_latency_ms:.6f} ms, "
-                    f"Power: {pimsim_power_mw:.6f} mW")
+                    f"Power: {pimsim_power_mw:.6f} mW, "
+                    f"Energy: {pimsim_energy_pj:.6f} pJ")
             except PimSimUnsupportedError as exc:
                 pimsim_status = PIMSIM_UNSUPPORTED
                 print_info(reporter, str(exc))
@@ -581,6 +585,7 @@ def validate_network(network_onnx_path, raptor_path, onnx_include_dir,
             pim_pass_timings=pim_pass_timings,
             pimsim_latency_ms=pimsim_latency_ms,
             pimsim_power_mw=pimsim_power_mw,
+            pimsim_energy_pj=pimsim_energy_pj,
             pimsim_status=pimsim_status,
         )
     except Exception: