finally fast googlenet with correct latency artifacts for fair comparison

validation/raptor_validation/pimsim_nn.py

@@ -0,0 +1,73 @@
+import json
+import re
+import shutil
+from pathlib import Path
+
+
+_METRIC_PATTERNS = {
+    "output_count": r"output count:\s+([0-9]+)\s+samples",
+    "throughput": r"throughput:\s+([0-9.eE+-]+)\s+samples/s",
+    "average_latency_ms": r"average latency:\s+([0-9.eE+-]+)\s+ms",
+    "latency_ms": r"latency:\s+([0-9.eE+-]+)\s+ms",
+    "average_power_mw": r"average power:\s+([0-9.eE+-]+)\s+mW",
+    "average_energy_pj": r"average energy:\s+([0-9.eE+-]+)\s+pJ(?:/it)?",
+}
+
+
+def parse_pimsim_nn_metrics(output):
+    metrics = {"raw_output": output}
+    for name, pattern in _METRIC_PATTERNS.items():
+        match = re.search(pattern, output)
+        if match:
+            value = match.group(1)
+            metrics[name] = int(value) if name == "output_count" else float(value)
+    return metrics
+
+
+def export_raptor_latency_artifact(pim_dir, output_dir):
+    pim_dir = Path(pim_dir)
+    output_dir = Path(output_dir)
+    if output_dir.exists():
+        shutil.rmtree(output_dir)
+    output_dir.mkdir(parents=True)
+
+    def int8_bytes(value, field):
+        if value % 4:
+            raise ValueError(f"Raptor {field}={value} is not aligned to its fp32 element width")
+        return value // 4
+
+    with open(pim_dir / "config.json", encoding="utf-8") as f:
+        config = json.load(f)
+    for field in ("inputs_addresses", "outputs_addresses"):
+        if field in config:
+            config[field] = [int8_bytes(value, field) for value in config[field]]
+    with open(output_dir / "config.json", "w", encoding="utf-8") as f:
+        json.dump(config, f, separators=(",", ":"))
+        f.write("\n")
+
+    byte_size_fields = {
+        "ld": "size",
+        "st": "size",
+        "lldi": "len",
+        "lmv": "len",
+        "send": "size",
+        "recv": "size",
+    }
+    for source in sorted(pim_dir.glob("core_*.json"), key=lambda path: int(path.stem.split("_")[1])):
+        with open(source, encoding="utf-8") as f:
+            instructions = json.load(f)
+        for instruction in instructions:
+            op = instruction["op"]
+            if op == "setbw":
+                instruction["ibiw"] = 8
+                instruction["obiw"] = 8
+            elif op == "sldi":
+                instruction["imm"] = int8_bytes(instruction["imm"], "address")
+            if field := byte_size_fields.get(op):
+                instruction[field] = int8_bytes(instruction[field], f"{op} {field}")
+                if offset := instruction.get("offset"):
+                    offset["offset_value"] = int8_bytes(offset["offset_value"], f"{op} offset")
+        with open(output_dir / source.name, "w", encoding="utf-8") as f:
+            json.dump(instructions, f, separators=(",", ":"))
+            f.write("\n")
+    return output_dir

validation/raptor_validation/validate_one.py

@@ -1,6 +1,5 @@
 import json
 import os
-import re
 import shutil
 import subprocess
 import sys
@@ -11,6 +10,7 @@ from colorama import Style, Fore
 from .gen_network_runner import gen_network_runner
 from .onnx_utils import gen_random_inputs, save_inputs_to_files, onnx_io, write_inputs_to_memory_bin, _ONNX_TO_NP
 from .raptor import compile_with_raptor
+from .pimsim_nn import export_raptor_latency_artifact, parse_pimsim_nn_metrics
 from .subprocess_utils import run_command_with_reporter
 
 STAGE_TITLES = (
@@ -61,9 +61,7 @@ PIMSIM_FAILED = "ERROR"
 PIMSIM_UNSUPPORTED = "UNSUPPORTED"
 PIMSIM_SKIPPED = "SKIP"
 PIMSIM_NOT_RUN = "-"
-PIMSIM_UNSUPPORTED_VSOFTMAX = "pimsim-nn does not support binary opcode vsoftmax"
-PIMSIM_LATENCY_RE = re.compile(r"\blatency:\s+([0-9.eE+-]+)\s+ms")
-PIMSIM_POWER_RE = re.compile(r"\baverage power:\s+([0-9.eE+-]+)\s+mW")
+PIMSIM_UNSUPPORTED_VSOFTMAX = "pimsim-nn does not support opcode vsoftmax"
 
 
 class PimSimUnsupportedError(RuntimeError):
@@ -80,6 +78,7 @@ class ValidationResult:
     pim_pass_timings: dict[str, float] = field(default_factory=dict)
     pimsim_latency_ms: float | None = None
     pimsim_power_mw: float | None = None
+    pimsim_energy_pj: float | None = None
     pimsim_status: str = PIMSIM_SKIPPED
 
 
@@ -262,9 +261,10 @@ def pimcomp_compatibility_errors(config_path, *, core_count, crossbar_count, cro
 
 
 def run_pimsim_nn(pimsim_nn_build_dir, pim_dir, config_path, reporter=None, timeout_sec=None):
+    latency_artifact = export_raptor_latency_artifact(pim_dir, Path(pim_dir).parent / "pimsim_nn")
     try:
         output = run_command(
-            [pimsim_nn_build_dir / "ChipTest", pim_dir, config_path, "--gui=false"],
+            [pimsim_nn_build_dir / "ChipTest", latency_artifact, config_path, "--gui=false"],
             cwd=pimsim_nn_build_dir,
             reporter=reporter,
             timeout_sec=timeout_sec,
@@ -275,11 +275,11 @@ def run_pimsim_nn(pimsim_nn_build_dir, pim_dir, config_path, reporter=None, time
         if PIMSIM_UNSUPPORTED_VSOFTMAX in error_output:
             raise PimSimUnsupportedError(PIMSIM_UNSUPPORTED_VSOFTMAX) from exc
         raise
-    latency_match = PIMSIM_LATENCY_RE.search(output)
-    power_match = PIMSIM_POWER_RE.search(output)
-    if not latency_match or not power_match:
-        raise RuntimeError("pimsim-nn output did not contain latency and average power")
-    return float(latency_match.group(1)), float(power_match.group(1))
+    metrics = parse_pimsim_nn_metrics(output)
+    required = ("latency_ms", "average_power_mw", "average_energy_pj")
+    if any(name not in metrics for name in required):
+        raise RuntimeError("pimsim-nn output did not contain latency, average power, and average energy")
+    return tuple(metrics[name] for name in required)
 
 
 def clean_workspace_artifacts(workspace_dir, model_stem):
@@ -416,6 +416,8 @@ def validate_network(network_onnx_path, raptor_path, onnx_include_dir,
         pimsim_nn_build_dir = Path(pimsim_nn_build_dir).resolve()
         pimsim_config_path = Path(pimsim_config_path).resolve()
     compile_extra_args = list(raptor_extra_args or [])
+    if pimsim_enabled and "--pim-emit-json" not in compile_extra_args:
+        compile_extra_args.append("--pim-emit-json")
     owns_reporter = reporter is None
     reporter = reporter or ProgressReporter(model_total, stages_per_model=len(MODE_STAGE_TITLES[mode]), verbose=verbose)
 
@@ -540,10 +542,11 @@ def validate_network(network_onnx_path, raptor_path, onnx_include_dir,
             "Run Non-functional Simulation")
         pimsim_latency_ms = None
         pimsim_power_mw = None
+        pimsim_energy_pj = None
         pimsim_status = PIMSIM_SKIPPED
         if pimsim_enabled:
             try:
-                pimsim_latency_ms, pimsim_power_mw = run_pimsim_nn(
+                pimsim_latency_ms, pimsim_power_mw, pimsim_energy_pj = run_pimsim_nn(
                     pimsim_nn_build_dir,
                     pim_dir,
                     pimsim_config_path,
@@ -554,7 +557,8 @@ def validate_network(network_onnx_path, raptor_path, onnx_include_dir,
                 print_info(
                     reporter,
                     f"Latency: {pimsim_latency_ms:.6f} ms, "
-                    f"Power: {pimsim_power_mw:.6f} mW")
+                    f"Power: {pimsim_power_mw:.6f} mW, "
+                    f"Energy: {pimsim_energy_pj:.6f} pJ")
             except PimSimUnsupportedError as exc:
                 pimsim_status = PIMSIM_UNSUPPORTED
                 print_info(reporter, str(exc))
@@ -581,6 +585,7 @@ def validate_network(network_onnx_path, raptor_path, onnx_include_dir,
             pim_pass_timings=pim_pass_timings,
             pimsim_latency_ms=pimsim_latency_ms,
             pimsim_power_mw=pimsim_power_mw,
+            pimsim_energy_pj=pimsim_energy_pj,
             pimsim_status=pimsim_status,
         )
     except Exception: