From 9053cc5e8b6a1992067fce2aefc982bc763015ed Mon Sep 17 00:00:00 2001 From: NiccoloN Date: Fri, 16 Oct 2026 17:57:55 +0200 Subject: [PATCH] minor validation tool fixes --- validation/tools/pim/ablation/README.md | 40 ++++++++++++------- validation/tools/pim/ablation/run_ablation.py | 24 +++-------- .../pim/pimcomp/compare/run_pimcomp_models.py | 2 + 3 files changed, 33 insertions(+), 33 deletions(-) diff --git a/validation/tools/pim/ablation/README.md b/validation/tools/pim/ablation/README.md index f9fe2aa..5f42a6c 100644 --- a/validation/tools/pim/ablation/README.md +++ b/validation/tools/pim/ablation/README.md @@ -23,9 +23,19 @@ latency and throughput pipeline 4. | `no-sync-no-plan` | Both options. | Every variant runs Raptor only. Pimcomp is not compiled, validated, or -simulated. Reference inputs and outputs are generated once under the shared -common-artifact root and reused by every variant. Ctrl+C terminates the active -variant and all of its worker jobs. +simulated; its validation column is `SKIPPED`, not `FAIL`. Reference inputs and +outputs are generated once under the shared common-artifact root and reused by +every variant. Ctrl+C terminates the active variant and all of its worker jobs. + +Removing synchronization deliberately removes host-buffer lifetime and pipeline +restart guarantees. The `no-sync` variants can therefore fail numerical +validation, especially across repeated throughput inputs. Those failures remain +`FAIL` and cause a nonzero exit; a completed ablation is not proof that every +variant is correct. Baseline and `no-plan` must still pass numerical validation. +The combined CSV retains `functional_validation` for every row. Performance +percentages from failing variants describe incorrect programs, not valid +speedups. Throughput deadline averages likewise do not establish a converged +steady-state period. The percentage baseline is `baseline`, so its available values are `+0.00%`. Every other @@ -38,7 +48,7 @@ values mean the metric is higher; negative values mean it is lower. | Option | Description and default | |---|---| | `-h`, `--help` | Show help and exit. | -| `--out-dir PATH` | Suite root. Default: `validation/networks/pimcomp_models`. Model artifacts are stored below `//artifacts`; disabled variants use `/[/pipelineN]/ablation/`. Variant summaries remain under `/ablation//`. | +| `--out-dir PATH` | Suite root. Default: `validation/networks/pimcomp_models`. Model artifacts are stored below `//artifacts//[/pipelineN]/ablation/`. Variant summaries remain under `/ablation//`. | | `--models MODEL [...]` | Models to run. Default: `vgg8 resnet18 resnet34 googlenet`; include `yolo11n` explicitly when needed. | | `--variant NAME` | Select a variant from the table above; repeat for multiple variants. Default: all variants. The feature-full baseline and percentage reference are added automatically when needed. | | `--dry-run` | Print the suite-runner commands that would run without modifying files. Default: off. | @@ -54,13 +64,15 @@ forwarded, the wrapper uses its default latency and throughput/pipeline-4 case set. Supplying either option overrides that default and is passed through as one suite-runner invocation per variant. -The feature-full `baseline` variant uses the normal `run_pimcomp_models.py` -artifact paths and is rerun with the same selected cases and forwarded options -as the disabled variants. The three disabled variants are stored below each model's -`artifacts//[/pipelineN]/ablation/` directory. Shared reference -artifacts remain under each model's `artifacts/common` directory. Transient -per-variant comparison summaries are removed after aggregation. The combined -table is written to `/results_ablation.csv`; it contains only the variant, case -identifiers, and signed percentage `latency`, `throughput`, `power`, and -`energy` columns. These are Raptor metrics; -Pimcomp metrics are omitted because the ablation invokes Raptor only. +The feature-full `baseline` variant is rerun with the same selected cases and +forwarded options as the disabled variants. All four variants are stored below +each model's `artifacts//[/pipelineN]/ablation/` directory. Shared +reference artifacts remain under each model's `artifacts/common` directory. +Normal comparison artifacts and summaries are preserved. Compact per-variant +comparison summaries remain under +`/ablation//results_comparison.csv` after aggregation. +The combined table is written to `/results_ablation.csv`; it contains the +variant, case identifiers, actual `functional_validation` status, and signed +percentage `latency`, `throughput`, `power`, and `energy` columns. These are +Raptor metrics; Pimcomp metrics are omitted because the ablation invokes Raptor +only. diff --git a/validation/tools/pim/ablation/run_ablation.py b/validation/tools/pim/ablation/run_ablation.py index 71ecf55..3b23c7e 100755 --- a/validation/tools/pim/ablation/run_ablation.py +++ b/validation/tools/pim/ablation/run_ablation.py @@ -52,7 +52,7 @@ PERCENTAGE_FIELDS = ( ("raptor_power_mw", "power"), ("raptor_energy_pj", "energy"), ) -RESULT_FIELDS = (*CASE_FIELDS, *(target for _, target in PERCENTAGE_FIELDS)) +RESULT_FIELDS = (*CASE_FIELDS, "functional_validation", *(target for _, target in PERCENTAGE_FIELDS)) DEFAULT_CASE_ARGUMENTS = ( ("latency", ("--mode", "latency")), ("throughput/pipeline4", ("--mode", "throughput", "--pipeline", "4")), @@ -106,20 +106,7 @@ def runner_argument_sets(forwarded: list[str]) -> tuple[tuple[str, list[str]], . def variant_output_dir(out_dir: Path, variant: Variant) -> Path: - return out_dir if variant.name == "baseline" else out_dir / "ablation" / variant.name - - -def remove_variant_summaries(out_dir: Path) -> None: - summary_root = out_dir / "ablation" - if not summary_root.is_dir(): - return - for summary in summary_root.glob("*/results_comparison.csv"): - summary.unlink() - for variant_dir in summary_root.iterdir(): - if variant_dir.is_dir() and not any(variant_dir.iterdir()): - variant_dir.rmdir() - if not any(summary_root.iterdir()): - summary_root.rmdir() + return out_dir / "ablation" / variant.name def runner_command( @@ -136,11 +123,11 @@ def runner_command( "--out-dir", str(out_dir), "--raptor-only", + "--ablation-variant", + variant.name, "--models", *models, ] - if variant.name != "baseline": - command.extend(("--ablation-variant", variant.name)) if not has_option(forwarded, "--common-dir"): command.extend(("--common-dir", str(common_root))) command.extend(forwarded) @@ -228,6 +215,7 @@ def aggregate_results( { "variant": variant.name, **{field: row.get(field, "") for field in CASE_FIELDS}, + "functional_validation": row.get("raptor_functional_validation", "NA"), **{ target: percentage_difference( row.get(source), reference.get(source) if reference else None @@ -301,7 +289,6 @@ def main(argv: list[str] | None = None) -> int: if current_process is not None: terminate_process_group(current_process) remove_lock_files(out_dir) - remove_variant_summaries(out_dir) print("Interrupted; terminated the active ablation job.", file=sys.stderr) return 130 @@ -313,7 +300,6 @@ def main(argv: list[str] | None = None) -> int: selected_cases, frozenset(args.models), ) - remove_variant_summaries(out_dir) failed.extend(aggregation_failures) if output is not None: print(f"Ablation results: {output}") diff --git a/validation/tools/pim/pimcomp/compare/run_pimcomp_models.py b/validation/tools/pim/pimcomp/compare/run_pimcomp_models.py index 115a62d..7dd1575 100755 --- a/validation/tools/pim/pimcomp/compare/run_pimcomp_models.py +++ b/validation/tools/pim/pimcomp/compare/run_pimcomp_models.py @@ -269,6 +269,8 @@ def comparison_passed(report: dict) -> bool: def functional_validation_status(result: dict | None) -> str: if result is None: return "NA" + if result.get("status") == "skipped": + return "SKIPPED" return "PASS" if result.get("status") == "done" and result.get("passed") else "FAIL"