diff --git a/validation/tools/pim/ablation/README.md b/validation/tools/pim/ablation/README.md index 3b9d054..f9fe2aa 100644 --- a/validation/tools/pim/ablation/README.md +++ b/validation/tools/pim/ablation/README.md @@ -18,17 +18,17 @@ latency and throughput pipeline 4. | Variant | Raptor options | |---|---| | `baseline` | None. | -| `no-synchronization` | `--pim-disable-synchronization` | -| `no-spatial-planning` | `--pim-disable-spatial-planning` | -| `no-synchronization-no-spatial-planning` | Both options. | +| `no-sync` | `--pim-disable-synchronization` | +| `no-plan` | `--pim-disable-spatial-planning` | +| `no-sync-no-plan` | Both options. | Every variant runs Raptor only. Pimcomp is not compiled, validated, or simulated. Reference inputs and outputs are generated once under the shared common-artifact root and reused by every variant. Ctrl+C terminates the active variant and all of its worker jobs. -The percentage baseline is `no-synchronization-no-spatial-planning`: both -ablation features are disabled, so its values are `+0.00%`. Every other +The percentage baseline is `baseline`, so its available values are `+0.00%`. +Every other variant reports the signed percentage difference of its Raptor metrics from that reference for the same model, architecture, mode, and pipeline. Positive values mean the metric is higher; negative values mean it is lower. @@ -61,6 +61,6 @@ as the disabled variants. The three disabled variants are stored below each mode artifacts remain under each model's `artifacts/common` directory. Transient per-variant comparison summaries are removed after aggregation. The combined table is written to `/results_ablation.csv`; it contains only the variant, case -identifiers, and signed `latency_percent`, `throughput_percent`, -`power_percent`, and `energy_percent` columns. These are Raptor metrics; +identifiers, and signed percentage `latency`, `throughput`, `power`, and +`energy` columns. These are Raptor metrics; Pimcomp metrics are omitted because the ablation invokes Raptor only. diff --git a/validation/tools/pim/ablation/run_ablation.py b/validation/tools/pim/ablation/run_ablation.py index 267af38..71ecf55 100755 --- a/validation/tools/pim/ablation/run_ablation.py +++ b/validation/tools/pim/ablation/run_ablation.py @@ -35,23 +35,22 @@ class Variant: VARIANTS = ( Variant("baseline", ()), - Variant("no-synchronization", ("--pim-disable-synchronization",)), - Variant("no-spatial-planning", ("--pim-disable-spatial-planning",)), + Variant("no-sync", ("--pim-disable-synchronization",)), + Variant("no-plan", ("--pim-disable-spatial-planning",)), Variant( - "no-synchronization-no-spatial-planning", + "no-sync-no-plan", ("--pim-disable-synchronization", "--pim-disable-spatial-planning"), ), ) VARIANT_BY_NAME = {variant.name: variant for variant in VARIANTS} -REFERENCE_VARIANT = "no-synchronization-no-spatial-planning" COMPARISON_RESULTS_FILENAME = "results_comparison.csv" ABLATION_RESULTS_FILENAME = "results_ablation.csv" CASE_FIELDS = ("arch", "model", "mode", "raptor_pipeline") PERCENTAGE_FIELDS = ( - ("raptor_latency_ms", "latency_percent"), - ("raptor_throughput_samples_s", "throughput_percent"), - ("raptor_power_mw", "power_percent"), - ("raptor_energy_pj", "energy_percent"), + ("raptor_latency_ms", "latency"), + ("raptor_throughput_samples_s", "throughput"), + ("raptor_power_mw", "power"), + ("raptor_energy_pj", "energy"), ) RESULT_FIELDS = (*CASE_FIELDS, *(target for _, target in PERCENTAGE_FIELDS)) DEFAULT_CASE_ARGUMENTS = ( @@ -90,7 +89,6 @@ def parse_args(argv: list[str] | None = None) -> tuple[argparse.Namespace, list[ def selected_variants(names: list[str] | None) -> list[Variant]: requested = set(names or VARIANT_BY_NAME) requested.add("baseline") - requested.add(REFERENCE_VARIANT) return [variant for variant in VARIANTS if variant.name in requested] @@ -215,31 +213,43 @@ def aggregate_results( selected_rows.append(row) if fields is None: return None, failures - reference_rows = { + baseline_rows = { tuple(row.get(field, "") for field in CASE_FIELDS): row - for row in rows_by_variant.get(REFERENCE_VARIANT, []) + for row in rows_by_variant.get("baseline", []) } - if REFERENCE_VARIANT not in rows_by_variant: - failures.append(f"{REFERENCE_VARIANT}: missing percentage reference results") + if "baseline" not in rows_by_variant: + failures.append("baseline: missing percentage reference results") + result_rows = [] + variant_order = {variant.name: index for index, variant in enumerate(VARIANTS)} + for variant in variants: + for row in rows_by_variant.get(variant.name, []): + reference = baseline_rows.get(tuple(row.get(field, "") for field in CASE_FIELDS)) + result_rows.append( + { + "variant": variant.name, + **{field: row.get(field, "") for field in CASE_FIELDS}, + **{ + target: percentage_difference( + row.get(source), reference.get(source) if reference else None + ) + for source, target in PERCENTAGE_FIELDS + }, + } + ) + result_rows.sort( + key=lambda row: ( + row["arch"], + row["model"], + row["mode"], + variant_order[row["variant"]], + int(row["raptor_pipeline"]), + ) + ) output = out_dir / ABLATION_RESULTS_FILENAME with output.open("w", newline="", encoding="utf-8") as stream: writer = csv.DictWriter(stream, fieldnames=("variant", *RESULT_FIELDS), lineterminator="\n") writer.writeheader() - for variant in variants: - for row in rows_by_variant.get(variant.name, []): - reference = reference_rows.get(tuple(row.get(field, "") for field in CASE_FIELDS)) - writer.writerow( - { - "variant": variant.name, - **{field: row.get(field, "") for field in CASE_FIELDS}, - **{ - target: percentage_difference( - row.get(source), reference.get(source) if reference else None - ) - for source, target in PERCENTAGE_FIELDS - }, - } - ) + writer.writerows(result_rows) return output, failures