From 2e76164aeddaa831a0cad53529dbdd820f0b4677 Mon Sep 17 00:00:00 2001 From: NiccoloN Date: Thu, 6 Aug 2026 21:49:54 +0200 Subject: [PATCH] more complete pimcomp comparison scripts update pimsim-nn submodule --- backend-simulators/pim/pimsim-nn | 2 +- validation/README.md | 34 +- validation/networks/pimcomp_models/README.md | 85 +- ...tency.onnx => googlenet-12-pimsim-nn.onnx} | Bin .../networks/pimcomp_models/results.csv | 46 +- .../pimcomp_models/validation_results.csv | 4 +- ...1n-latency.onnx => yolo11n-pimsim-nn.onnx} | Bin ...fig.json => throughput_config_1000ms.json} | 0 .../arch-a/throughput_config_100ms.json | 61 ++ ...fig.json => throughput_config_1000ms.json} | 0 .../arch-b/throughput_config_100ms.json | 61 ++ ...fig.json => throughput_config_1000ms.json} | 0 .../arch-c/throughput_config_100ms.json | 61 ++ .../compare}/compare_raptor_pimcomp.py | 363 +++++---- .../compare/run_pimcomp_paper_latency.py | 755 ++++++++++++++++++ .../tools/pim/pimcomp/correctness/README.md | 49 ++ .../correctness/prefill_batch_memory.py | 171 ++++ .../correctness/run_prefill_experiment.py | 127 +++ .../{ => pim/visuals}/pim_sequence_diagram.py | 0 .../pimcomp/run_pimcomp_paper_latency.py | 355 -------- 20 files changed, 1626 insertions(+), 548 deletions(-) rename validation/networks/pimcomp_models/googlenet/{googlenet-12-latency.onnx => googlenet-12-pimsim-nn.onnx} (100%) rename validation/networks/pimcomp_models/yolo11n/{yolo11n-latency.onnx => yolo11n-pimsim-nn.onnx} (100%) rename validation/pimsim_configs/pimcomp/arch-a/{throughput_config.json => throughput_config_1000ms.json} (100%) create mode 100644 validation/pimsim_configs/pimcomp/arch-a/throughput_config_100ms.json rename validation/pimsim_configs/pimcomp/arch-b/{throughput_config.json => throughput_config_1000ms.json} (100%) create mode 100644 validation/pimsim_configs/pimcomp/arch-b/throughput_config_100ms.json rename validation/pimsim_configs/pimcomp/arch-c/{throughput_config.json => throughput_config_1000ms.json} (100%) create mode 100644 validation/pimsim_configs/pimcomp/arch-c/throughput_config_100ms.json rename validation/tools/{pimcomp => pim/pimcomp/compare}/compare_raptor_pimcomp.py (87%) create mode 100755 validation/tools/pim/pimcomp/compare/run_pimcomp_paper_latency.py create mode 100644 validation/tools/pim/pimcomp/correctness/README.md create mode 100644 validation/tools/pim/pimcomp/correctness/prefill_batch_memory.py create mode 100755 validation/tools/pim/pimcomp/correctness/run_prefill_experiment.py rename validation/tools/{ => pim/visuals}/pim_sequence_diagram.py (100%) delete mode 100755 validation/tools/pimcomp/run_pimcomp_paper_latency.py diff --git a/backend-simulators/pim/pimsim-nn b/backend-simulators/pim/pimsim-nn index 6a38325..f7446df 160000 --- a/backend-simulators/pim/pimsim-nn +++ b/backend-simulators/pim/pimsim-nn @@ -1 +1 @@ -Subproject commit 6a3832525b7e0a68ddbcde87f066f6d854085215 +Subproject commit f7446dfc921a9885e7269fae206227880b2a303c diff --git a/validation/README.md b/validation/README.md index 34a61ab..6824118 100644 --- a/validation/README.md +++ b/validation/README.md @@ -63,22 +63,24 @@ that directory while retaining the terminal table. ## Raptor vs PIMCOMP comparison -The PIMCOMP paper-model suite has a one-command Arch-A comparison: +The PIMCOMP paper-model suite has a one-command multi-architecture comparison: ```bash -.venv/bin/python validation/tools/pimcomp/run_pimcomp_paper_latency.py +.venv/bin/python validation/tools/pim/pimcomp/compare/run_pimcomp_paper_latency.py ``` The runner verifies PIMCOMP's population-200, 1000-iteration GA settings, builds Raptor and the existing `third_party/PIMCOMP-NN/build` tree, and compares -the four paper models one at a time. Each PIMCOMP GA run evaluates candidates -in parallel; set `OMP_NUM_THREADS` to control its worker count. Use +the four paper models in parallel. Set `--jobs` to control comparison workers; +the runner leaves `OMP_NUM_THREADS` at its environment default. Use `--models vgg8` for one model or `--dry-run` to print the commands. Generated artifacts are stored beside each model and ignored by Git. The -comparison reuses the same model-level `inputs/`, `outputs/`, `runner/`, -`raptor/`, and `simulation/` paths as regular validation. PIMCOMP-only -artifacts and `comparison_report.{md,json}` live under `pimcomp/`. See +comparison reuses model-level `common/inputs/`, `common/outputs/`, +`common/runner/` across architectures, modes, and pipelines. Pimsim configs and +network meshes are referenced directly from +`validation/pimsim_configs/pimcomp/`. Architecture- and pipeline-specific `raptor/`, `simulation/`, and +PIMCOMP artifacts stay under each comparison directory. See [`networks/pimcomp_models/README.md`](networks/pimcomp_models/README.md) for profiles, model provenance, limitations, and remote execution. @@ -164,7 +166,9 @@ The selected PIMCOMP profile must match `--core-count`, `--crossbar-count`, and prints the incompatible values; functional validation still runs. The checked-in profiles are under -`validation/pimsim_configs/pimcomp//latency_config.json`. +`validation/pimsim_configs/pimcomp//`; latency uses +`latency_config.json`, while throughput uses +`throughput_config_