From eafd96fd2d887e09704f79357ab6b0c725f1a886 Mon Sep 17 00:00:00 2001 From: NiccoloN Date: Mon, 24 Aug 2026 12:21:35 +0200 Subject: [PATCH] refactor experiments document sync experiment --- .../tools/pim/pimcomp/correctness/README.md | 51 +++++++++++++++++-- .../batch_memory.py} | 0 .../run_experiment.py} | 4 +- .../synchronization/architecture_contract.py} | 10 ++-- .../architecture_evidence.json} | 0 .../synchronization/global_memory.py} | 6 +-- .../run_experiment.py} | 16 +++--- 7 files changed, 65 insertions(+), 22 deletions(-) rename validation/tools/pim/pimcomp/correctness/{prefill_batch_memory.py => prefill/batch_memory.py} (100%) rename validation/tools/pim/pimcomp/correctness/{run_prefill_experiment.py => prefill/run_experiment.py} (98%) rename validation/tools/pim/pimcomp/{compare/test_PIMCOMP_architecture_sync_contract.py => correctness/synchronization/architecture_contract.py} (96%) rename validation/tools/pim/pimcomp/{compare/pimcomp_architecture_sync_evidence.json => correctness/synchronization/architecture_evidence.json} (100%) rename validation/tools/pim/pimcomp/{compare/test_PIMCOMP_global_memory_sync.py => correctness/synchronization/global_memory.py} (99%) rename validation/tools/pim/pimcomp/correctness/{test_PIMCOMP_adversarial_memory_sync.py => synchronization/run_experiment.py} (99%) diff --git a/validation/tools/pim/pimcomp/correctness/README.md b/validation/tools/pim/pimcomp/correctness/README.md index 27d2702..0df2a73 100644 --- a/validation/tools/pim/pimcomp/correctness/README.md +++ b/validation/tools/pim/pimcomp/correctness/README.md @@ -1,4 +1,47 @@ -# Pimcomp batch correctness reproduction +# Pimcomp correctness experiments + +## Synchronization ordering + +The synchronization experiment checks whether repeated PIMCOMP and Raptor +programs preserve cross-core global-memory generations under legal execution +schedules. It builds a small two-convolution model for Arch-A, Arch-B, and +Arch-C, finds cross-core `ST`/`LD` dependencies, and compares greedy, +bounded-stall, randomized, and adversarial runs using the Rust simulator's +provenance trace. Byte-identical batch inputs isolate intermediate-memory +ordering from the separate host-input lifetime issue. + +The experiment records static dependency evidence, dynamic provenance, +functional output comparisons, architecture contract classifications, and +per-artifact PIMCOMP/Raptor conclusions. Diagnostic scheduling changes only +the functional simulator's execution order; it does not modify PIMCOMP or the +`pimsim-nn` performance oracle. + +Prerequisites are the repository virtual environment, the release Raptor +compiler, built PIMCOMP frontend/backend, and the existing `pimsim-nn` build. +Run from the repository root with a new or empty output directory: + +```bash +.venv/bin/python \ + validation/tools/pim/pimcomp/correctness/synchronization/run_experiment.py \ + --out-dir /tmp/pimcomp-adversarial-sync \ + --batch-size 4 --seed 0 --self-check +``` + +The directory contains: + +- [`run_experiment.py`](synchronization/run_experiment.py): orchestration, + classification, reporting, and self-checks. +- [`global_memory.py`](synchronization/global_memory.py): artifact compilation, + dependency analysis, simulator execution, and provenance helpers. +- [`architecture_contract.py`](synchronization/architecture_contract.py) and + [`architecture_evidence.json`](synchronization/architecture_evidence.json): + conservative architecture-contract evidence and labels. + +The main outputs are `adversarial_memory_sync_report.json` and +`adversarial_memory_sync_report.md`, with per-architecture evidence below the +same output root. + +## Pimcomp batch prefill reproduction Pimcomp's batch scheduler currently emits an incomplete standalone program for models containing post operations. The generated `VerificationInfo.json` uses a @@ -24,14 +67,14 @@ schedule. Run the default reproduction from the repository root: ```bash -.venv/bin/python validation/tools/pim/pimcomp/correctness/run_prefill_experiment.py +.venv/bin/python validation/tools/pim/pimcomp/correctness/prefill/run_experiment.py ``` The launcher accepts an alternate comparison directory, model, work directory, and shared reference-artifact directory: ```bash -.venv/bin/python validation/tools/pim/pimcomp/correctness/run_prefill_experiment.py \ +.venv/bin/python validation/tools/pim/pimcomp/correctness/prefill/run_experiment.py \ validation/networks/pimcomp_models/googlenet/artifacts/arch-a/throughput/pipeline2 \ validation/networks/pimcomp_models/googlenet/googlenet-12-pimsim-nn.onnx \ validation/networks/pimcomp_models/googlenet/artifacts/arch-a/throughput/pipeline2/correctness/prefill \ @@ -42,7 +85,7 @@ Without the optional work-directory argument, the experiment uses the same `correctness/prefill/` directory below the comparison artifacts. It runs the exported artifact once with its original memory image and once -with [`prefill_batch_memory.py`](prefill_batch_memory.py), then compares both +with [`batch_memory.py`](prefill/batch_memory.py), then compares both outputs with the recorded native reference. The expected GoogLeNet result is a baseline maximum difference near `6.70705` and a prefilled maximum difference near `4.05e-6`. diff --git a/validation/tools/pim/pimcomp/correctness/prefill_batch_memory.py b/validation/tools/pim/pimcomp/correctness/prefill/batch_memory.py similarity index 100% rename from validation/tools/pim/pimcomp/correctness/prefill_batch_memory.py rename to validation/tools/pim/pimcomp/correctness/prefill/batch_memory.py diff --git a/validation/tools/pim/pimcomp/correctness/run_prefill_experiment.py b/validation/tools/pim/pimcomp/correctness/prefill/run_experiment.py similarity index 98% rename from validation/tools/pim/pimcomp/correctness/run_prefill_experiment.py rename to validation/tools/pim/pimcomp/correctness/prefill/run_experiment.py index 04da777..ae81ccd 100755 --- a/validation/tools/pim/pimcomp/correctness/run_prefill_experiment.py +++ b/validation/tools/pim/pimcomp/correctness/prefill/run_experiment.py @@ -11,11 +11,11 @@ from pathlib import Path import numpy as np -from prefill_batch_memory import prefill_batch_memory +from batch_memory import prefill_batch_memory SCRIPT_DIR = Path(__file__).resolve().parent -REPO_ROOT = SCRIPT_DIR.parents[4] +REPO_ROOT = SCRIPT_DIR.parents[5] DEFAULT_COMPARISON_DIR = REPO_ROOT / "validation/networks/pimcomp_models/googlenet/artifacts/arch-a/throughput/pipeline2" DEFAULT_MODEL = REPO_ROOT / "validation/networks/pimcomp_models/googlenet/googlenet-12-pimsim-nn.onnx" DEFAULT_COMMON_DIR = REPO_ROOT / "validation/networks/pimcomp_models/googlenet/artifacts/common" diff --git a/validation/tools/pim/pimcomp/compare/test_PIMCOMP_architecture_sync_contract.py b/validation/tools/pim/pimcomp/correctness/synchronization/architecture_contract.py similarity index 96% rename from validation/tools/pim/pimcomp/compare/test_PIMCOMP_architecture_sync_contract.py rename to validation/tools/pim/pimcomp/correctness/synchronization/architecture_contract.py index 3169e82..3e7c9fe 100644 --- a/validation/tools/pim/pimcomp/compare/test_PIMCOMP_architecture_sync_contract.py +++ b/validation/tools/pim/pimcomp/correctness/synchronization/architecture_contract.py @@ -15,12 +15,13 @@ from pathlib import Path from typing import Any -REPO = Path(__file__).resolve().parents[5] +REPO = Path(__file__).resolve().parents[6] VALIDATION = REPO / "validation" PIMCOMP_ROOT = REPO / "third_party/PIMCOMP-NN" PIMSIM_ROOT = REPO / "backend-simulators/pim/pimsim-nn" RUST_ROOT = REPO / "backend-simulators/pim/pim-simulator" -EVIDENCE_PATH = Path(__file__).with_name("pimcomp_architecture_sync_evidence.json") +COMPARE_DIR = REPO / "validation/tools/pim/pimcomp/compare" +EVIDENCE_PATH = Path(__file__).with_name("architecture_evidence.json") def _default_evidence() -> dict[str, Any]: @@ -161,9 +162,8 @@ def make_identical_inputs(model: Path, batch_size: int, out: Path) -> list[Path] import sys - compare_dir = Path(__file__).resolve().parent - if str(compare_dir) not in sys.path: - sys.path.insert(0, str(compare_dir)) + if str(COMPARE_DIR) not in sys.path: + sys.path.insert(0, str(COMPARE_DIR)) import compare_raptor_pimcomp_model as compare # noqa: PLC0415 inputs, _ = compare.onnx_io(model) diff --git a/validation/tools/pim/pimcomp/compare/pimcomp_architecture_sync_evidence.json b/validation/tools/pim/pimcomp/correctness/synchronization/architecture_evidence.json similarity index 100% rename from validation/tools/pim/pimcomp/compare/pimcomp_architecture_sync_evidence.json rename to validation/tools/pim/pimcomp/correctness/synchronization/architecture_evidence.json diff --git a/validation/tools/pim/pimcomp/compare/test_PIMCOMP_global_memory_sync.py b/validation/tools/pim/pimcomp/correctness/synchronization/global_memory.py similarity index 99% rename from validation/tools/pim/pimcomp/compare/test_PIMCOMP_global_memory_sync.py rename to validation/tools/pim/pimcomp/correctness/synchronization/global_memory.py index 2342d7a..a1d4057 100644 --- a/validation/tools/pim/pimcomp/compare/test_PIMCOMP_global_memory_sync.py +++ b/validation/tools/pim/pimcomp/correctness/synchronization/global_memory.py @@ -16,17 +16,17 @@ from typing import Any import numpy as np -REPO = Path(__file__).resolve().parents[5] +REPO = Path(__file__).resolve().parents[6] VALIDATION = REPO / "validation" CONFIG_ROOT = VALIDATION / "pimsim_configs/pimcomp" PIMCOMP_ROOT = REPO / "third_party/PIMCOMP-NN" PIMSIM_NN_ROOT = REPO / "backend-simulators/pim/pimsim-nn" RUST_ROOT = REPO / "backend-simulators/pim/pim-simulator" RUST_BINARY = RUST_ROOT / "target/release/pim-simulator" -COMPARE_SCRIPT = Path(__file__).with_name("compare_raptor_pimcomp_model.py") +COMPARE_SCRIPT = REPO / "validation/tools/pim/pimcomp/compare/compare_raptor_pimcomp_model.py" PYTHON = REPO / ".venv/bin/python" -sys.path.insert(0, str(Path(__file__).resolve().parent)) +sys.path.insert(0, str(COMPARE_SCRIPT.parent)) import compare_raptor_pimcomp_model as compare # noqa: E402 diff --git a/validation/tools/pim/pimcomp/correctness/test_PIMCOMP_adversarial_memory_sync.py b/validation/tools/pim/pimcomp/correctness/synchronization/run_experiment.py similarity index 99% rename from validation/tools/pim/pimcomp/correctness/test_PIMCOMP_adversarial_memory_sync.py rename to validation/tools/pim/pimcomp/correctness/synchronization/run_experiment.py index 2a60b30..4b3e961 100644 --- a/validation/tools/pim/pimcomp/correctness/test_PIMCOMP_adversarial_memory_sync.py +++ b/validation/tools/pim/pimcomp/correctness/synchronization/run_experiment.py @@ -3,7 +3,7 @@ One documented invocation is: - .venv/bin/python validation/tools/pim/pimcomp/correctness/test_PIMCOMP_adversarial_memory_sync.py \ + .venv/bin/python validation/tools/pim/pimcomp/correctness/synchronization/run_experiment.py \ --out-dir /tmp/pimcomp-adversarial-sync --batch-size 4 --seed 0 --self-check The experiment uses identical external input bytes for throughput runs so the @@ -24,16 +24,16 @@ from pathlib import Path from typing import Any -REPO = Path(__file__).resolve().parents[5] +REPO = Path(__file__).resolve().parents[6] SCRIPT = Path(__file__).resolve() -HELPER_DIR = REPO / "validation/tools/pim/pimcomp/compare" -GLOBAL_SCRIPT = HELPER_DIR / "test_PIMCOMP_global_memory_sync.py" -AUDIT_SCRIPT = HELPER_DIR / "test_PIMCOMP_architecture_sync_contract.py" +EXPERIMENT_DIR = SCRIPT.parent +GLOBAL_SCRIPT = EXPERIMENT_DIR / "global_memory.py" +AUDIT_SCRIPT = EXPERIMENT_DIR / "architecture_contract.py" PYTHON = REPO / ".venv/bin/python" -sys.path.insert(0, str(HELPER_DIR)) -import test_PIMCOMP_architecture_sync_contract as audit # noqa: E402 -import test_PIMCOMP_global_memory_sync as global_sync # noqa: E402 +sys.path.insert(0, str(EXPERIMENT_DIR)) +import architecture_contract as audit # noqa: E402 +import global_memory as global_sync # noqa: E402 INVALID = "INVALID_SYNCHRONIZATION_REPRODUCER"