minor validation tool fixes

validation/tools/pim/ablation/README.md

@@ -23,9 +23,19 @@ latency and throughput pipeline 4.
 | `no-sync-no-plan` | Both options. |
 
 Every variant runs Raptor only. Pimcomp is not compiled, validated, or
-simulated. Reference inputs and outputs are generated once under the shared
+simulated; its validation column is `SKIPPED`, not `FAIL`. Reference inputs and
-common-artifact root and reused by every variant. Ctrl+C terminates the active
+outputs are generated once under the shared common-artifact root and reused by
-variant and all of its worker jobs.
+every variant. Ctrl+C terminates the active variant and all of its worker jobs.
+
+Removing synchronization deliberately removes host-buffer lifetime and pipeline
+restart guarantees. The `no-sync` variants can therefore fail numerical
+validation, especially across repeated throughput inputs. Those failures remain
+`FAIL` and cause a nonzero exit; a completed ablation is not proof that every
+variant is correct. Baseline and `no-plan` must still pass numerical validation.
+The combined CSV retains `functional_validation` for every row. Performance
+percentages from failing variants describe incorrect programs, not valid
+speedups. Throughput deadline averages likewise do not establish a converged
+steady-state period.
 
 The percentage baseline is `baseline`, so its available values are `+0.00%`.
 Every other
@@ -38,7 +48,7 @@ values mean the metric is higher; negative values mean it is lower.
 | Option | Description and default |
 |---|---|
 | `-h`, `--help` | Show help and exit. |
-| `--out-dir PATH` | Suite root. Default: `validation/networks/pimcomp_models`. Model artifacts are stored below `<out-dir>/<model>/artifacts`; disabled variants use `<arch>/<mode>[/pipelineN]/ablation/<variant>`. Variant summaries remain under `<out-dir>/ablation/<variant>/`. |
+| `--out-dir PATH` | Suite root. Default: `validation/networks/pimcomp_models`. Model artifacts are stored below `<out-dir>/<model>/artifacts/<arch>/<mode>[/pipelineN]/ablation/<variant>`. Variant summaries remain under `<out-dir>/ablation/<variant>/`. |
 | `--models MODEL [...]` | Models to run. Default: `vgg8 resnet18 resnet34 googlenet`; include `yolo11n` explicitly when needed. |
 | `--variant NAME` | Select a variant from the table above; repeat for multiple variants. Default: all variants. The feature-full baseline and percentage reference are added automatically when needed. |
 | `--dry-run` | Print the suite-runner commands that would run without modifying files. Default: off. |
@@ -54,13 +64,15 @@ forwarded, the wrapper uses its default latency and throughput/pipeline-4
 case set. Supplying either option overrides that default and is passed through
 as one suite-runner invocation per variant.
 
-The feature-full `baseline` variant uses the normal `run_pimcomp_models.py`
+The feature-full `baseline` variant is rerun with the same selected cases and
-artifact paths and is rerun with the same selected cases and forwarded options
+forwarded options as the disabled variants. All four variants are stored below
-as the disabled variants. The three disabled variants are stored below each model's
+each model's `artifacts/<arch>/<mode>[/pipelineN]/ablation/` directory. Shared
-`artifacts/<arch>/<mode>[/pipelineN]/ablation/` directory. Shared reference
+reference artifacts remain under each model's `artifacts/common` directory.
-artifacts remain under each model's `artifacts/common` directory. Transient
+Normal comparison artifacts and summaries are preserved. Compact per-variant
-per-variant comparison summaries are removed after aggregation. The combined
+comparison summaries remain under
-table is written to `<out-dir>/results_ablation.csv`; it contains only the variant, case
+`<out-dir>/ablation/<variant>/results_comparison.csv` after aggregation.
-identifiers, and signed percentage `latency`, `throughput`, `power`, and
+The combined table is written to `<out-dir>/results_ablation.csv`; it contains the
-`energy` columns. These are Raptor metrics;
+variant, case identifiers, actual `functional_validation` status, and signed
-Pimcomp metrics are omitted because the ablation invokes Raptor only.
+percentage `latency`, `throughput`, `power`, and `energy` columns. These are
+Raptor metrics; Pimcomp metrics are omitted because the ablation invokes Raptor
+only.

validation/tools/pim/ablation/run_ablation.py

@@ -52,7 +52,7 @@ PERCENTAGE_FIELDS = (
     ("raptor_power_mw", "power"),
     ("raptor_energy_pj", "energy"),
 )
-RESULT_FIELDS = (*CASE_FIELDS, *(target for _, target in PERCENTAGE_FIELDS))
+RESULT_FIELDS = (*CASE_FIELDS, "functional_validation", *(target for _, target in PERCENTAGE_FIELDS))
 DEFAULT_CASE_ARGUMENTS = (
     ("latency", ("--mode", "latency")),
     ("throughput/pipeline4", ("--mode", "throughput", "--pipeline", "4")),
@@ -106,20 +106,7 @@ def runner_argument_sets(forwarded: list[str]) -> tuple[tuple[str, list[str]], .
 
 
 def variant_output_dir(out_dir: Path, variant: Variant) -> Path:
-    return out_dir if variant.name == "baseline" else out_dir / "ablation" / variant.name
+    return out_dir / "ablation" / variant.name
-
-
-def remove_variant_summaries(out_dir: Path) -> None:
-    summary_root = out_dir / "ablation"
-    if not summary_root.is_dir():
-        return
-    for summary in summary_root.glob("*/results_comparison.csv"):
-        summary.unlink()
-    for variant_dir in summary_root.iterdir():
-        if variant_dir.is_dir() and not any(variant_dir.iterdir()):
-            variant_dir.rmdir()
-    if not any(summary_root.iterdir()):
-        summary_root.rmdir()
 
 
 def runner_command(
@@ -136,11 +123,11 @@ def runner_command(
         "--out-dir",
         str(out_dir),
         "--raptor-only",
+        "--ablation-variant",
+        variant.name,
         "--models",
         *models,
     ]
-    if variant.name != "baseline":
-        command.extend(("--ablation-variant", variant.name))
     if not has_option(forwarded, "--common-dir"):
         command.extend(("--common-dir", str(common_root)))
     command.extend(forwarded)
@@ -228,6 +215,7 @@ def aggregate_results(
                 {
                     "variant": variant.name,
                     **{field: row.get(field, "") for field in CASE_FIELDS},
+                    "functional_validation": row.get("raptor_functional_validation", "NA"),
                     **{
                         target: percentage_difference(
                             row.get(source), reference.get(source) if reference else None
@@ -301,7 +289,6 @@ def main(argv: list[str] | None = None) -> int:
         if current_process is not None:
             terminate_process_group(current_process)
         remove_lock_files(out_dir)
-        remove_variant_summaries(out_dir)
         print("Interrupted; terminated the active ablation job.", file=sys.stderr)
         return 130
 
@@ -313,7 +300,6 @@ def main(argv: list[str] | None = None) -> int:
         selected_cases,
         frozenset(args.models),
     )
-    remove_variant_summaries(out_dir)
     failed.extend(aggregation_failures)
     if output is not None:
         print(f"Ablation results: {output}")

validation/tools/pim/pimcomp/compare/run_pimcomp_models.py

@@ -269,6 +269,8 @@ def comparison_passed(report: dict) -> bool:
 def functional_validation_status(result: dict | None) -> str:
     if result is None:
         return "NA"
+    if result.get("status") == "skipped":
+        return "SKIPPED"
     return "PASS" if result.get("status") == "done" and result.get("passed") else "FAIL"