fix ablation study

validation/tools/pim/ablation/README.md

@@ -18,17 +18,17 @@ latency and throughput pipeline 4.
 | Variant | Raptor options |
 |---|---|
 | `baseline` | None. |
-| `no-synchronization` | `--pim-disable-synchronization` |
+| `no-sync` | `--pim-disable-synchronization` |
-| `no-spatial-planning` | `--pim-disable-spatial-planning` |
+| `no-plan` | `--pim-disable-spatial-planning` |
-| `no-synchronization-no-spatial-planning` | Both options. |
+| `no-sync-no-plan` | Both options. |
 
 Every variant runs Raptor only. Pimcomp is not compiled, validated, or
 simulated. Reference inputs and outputs are generated once under the shared
 common-artifact root and reused by every variant. Ctrl+C terminates the active
 variant and all of its worker jobs.
 
-The percentage baseline is `no-synchronization-no-spatial-planning`: both
+The percentage baseline is `baseline`, so its available values are `+0.00%`.
-ablation features are disabled, so its values are `+0.00%`. Every other
+Every other
 variant reports the signed percentage difference of its Raptor metrics from
 that reference for the same model, architecture, mode, and pipeline. Positive
 values mean the metric is higher; negative values mean it is lower.
@@ -61,6 +61,6 @@ as the disabled variants. The three disabled variants are stored below each mode
 artifacts remain under each model's `artifacts/common` directory. Transient
 per-variant comparison summaries are removed after aggregation. The combined
 table is written to `<out-dir>/results_ablation.csv`; it contains only the variant, case
-identifiers, and signed `latency_percent`, `throughput_percent`,
+identifiers, and signed percentage `latency`, `throughput`, `power`, and
-`power_percent`, and `energy_percent` columns. These are Raptor metrics;
+`energy` columns. These are Raptor metrics;
 Pimcomp metrics are omitted because the ablation invokes Raptor only.

validation/tools/pim/ablation/run_ablation.py

@@ -35,23 +35,22 @@ class Variant:
 
 VARIANTS = (
     Variant("baseline", ()),
-    Variant("no-synchronization", ("--pim-disable-synchronization",)),
+    Variant("no-sync", ("--pim-disable-synchronization",)),
-    Variant("no-spatial-planning", ("--pim-disable-spatial-planning",)),
+    Variant("no-plan", ("--pim-disable-spatial-planning",)),
     Variant(
-        "no-synchronization-no-spatial-planning",
+        "no-sync-no-plan",
         ("--pim-disable-synchronization", "--pim-disable-spatial-planning"),
     ),
 )
 VARIANT_BY_NAME = {variant.name: variant for variant in VARIANTS}
-REFERENCE_VARIANT = "no-synchronization-no-spatial-planning"
 COMPARISON_RESULTS_FILENAME = "results_comparison.csv"
 ABLATION_RESULTS_FILENAME = "results_ablation.csv"
 CASE_FIELDS = ("arch", "model", "mode", "raptor_pipeline")
 PERCENTAGE_FIELDS = (
-    ("raptor_latency_ms", "latency_percent"),
+    ("raptor_latency_ms", "latency"),
-    ("raptor_throughput_samples_s", "throughput_percent"),
+    ("raptor_throughput_samples_s", "throughput"),
-    ("raptor_power_mw", "power_percent"),
+    ("raptor_power_mw", "power"),
-    ("raptor_energy_pj", "energy_percent"),
+    ("raptor_energy_pj", "energy"),
 )
 RESULT_FIELDS = (*CASE_FIELDS, *(target for _, target in PERCENTAGE_FIELDS))
 DEFAULT_CASE_ARGUMENTS = (
@@ -90,7 +89,6 @@ def parse_args(argv: list[str] | None = None) -> tuple[argparse.Namespace, list[
 def selected_variants(names: list[str] | None) -> list[Variant]:
     requested = set(names or VARIANT_BY_NAME)
     requested.add("baseline")
-    requested.add(REFERENCE_VARIANT)
     return [variant for variant in VARIANTS if variant.name in requested]
 
 
@@ -215,31 +213,43 @@ def aggregate_results(
                 selected_rows.append(row)
     if fields is None:
         return None, failures
-    reference_rows = {
+    baseline_rows = {
         tuple(row.get(field, "") for field in CASE_FIELDS): row
-        for row in rows_by_variant.get(REFERENCE_VARIANT, [])
+        for row in rows_by_variant.get("baseline", [])
     }
-    if REFERENCE_VARIANT not in rows_by_variant:
+    if "baseline" not in rows_by_variant:
-        failures.append(f"{REFERENCE_VARIANT}: missing percentage reference results")
+        failures.append("baseline: missing percentage reference results")
+    result_rows = []
+    variant_order = {variant.name: index for index, variant in enumerate(VARIANTS)}
+    for variant in variants:
+        for row in rows_by_variant.get(variant.name, []):
+            reference = baseline_rows.get(tuple(row.get(field, "") for field in CASE_FIELDS))
+            result_rows.append(
+                {
+                    "variant": variant.name,
+                    **{field: row.get(field, "") for field in CASE_FIELDS},
+                    **{
+                        target: percentage_difference(
+                            row.get(source), reference.get(source) if reference else None
+                        )
+                        for source, target in PERCENTAGE_FIELDS
+                    },
+                }
+            )
+    result_rows.sort(
+        key=lambda row: (
+            row["arch"],
+            row["model"],
+            row["mode"],
+            variant_order[row["variant"]],
+            int(row["raptor_pipeline"]),
+        )
+    )
     output = out_dir / ABLATION_RESULTS_FILENAME
     with output.open("w", newline="", encoding="utf-8") as stream:
         writer = csv.DictWriter(stream, fieldnames=("variant", *RESULT_FIELDS), lineterminator="\n")
         writer.writeheader()
-        for variant in variants:
+        writer.writerows(result_rows)
-            for row in rows_by_variant.get(variant.name, []):
-                reference = reference_rows.get(tuple(row.get(field, "") for field in CASE_FIELDS))
-                writer.writerow(
-                    {
-                        "variant": variant.name,
-                        **{field: row.get(field, "") for field in CASE_FIELDS},
-                        **{
-                            target: percentage_difference(
-                                row.get(source), reference.get(source) if reference else None
-                            )
-                            for source, target in PERCENTAGE_FIELDS
-                        },
-                    }
-                )
     return output, failures