better compile times for throughput scheduling
Validate Operations / validate-operations (push) Has been cancelled

fix failing no-plan throughput ablation runs
update results
This commit is contained in:
NiccoloN
2026-10-28 16:34:48 +01:00
parent 3c7679594b
commit 5f2db946f0
28 changed files with 909 additions and 197 deletions
@@ -1,41 +1,41 @@
arch,model,mode,raptor_pipeline,pimcomp_pipeline,raptor_functional_validation,pimcomp_functional_validation,raptor_latency_ms,pimcomp_latency_ms,raptor_throughput_samples_s,pimcomp_throughput_samples_s,raptor_power_mw,pimcomp_power_mw,raptor_energy_pj,pimcomp_energy_pj,better_compiler,speedup
arch-a,vgg8,latency,1,element,PASS,PASS,1.51833,7.98507,NA,NA,321.046,200.111,4.87455e+08,1.5979e+09,raptor,5.26
arch-a,vgg8,throughput,2,batch,PASS,PASS,0.906321,1.0364,1100,965,429.311,482.597,4.07972e+08,5.41692e+08,raptor,1.14
arch-a,vgg8,throughput,4,batch,PASS,PASS,0.778242,1.0364,1280,965,457.039,482.597,4.15762e+08,5.41692e+08,raptor,1.33
arch-a,vgg8,throughput,8,batch,PASS,PASS,0.778094,1.0364,1290,965,464.071,482.597,3.96424e+08,5.41692e+08,raptor,1.34
arch-a,resnet18,latency,1,element,PASS,FAIL,14.0332,58.1775,NA,NA,455.67,238.389,6.3945e+09,1.38689e+10,raptor,4.15
arch-a,resnet18,throughput,2,batch,PASS,FAIL,11.7314,13.4585,85.2,74.3,503.111,481.604,6.62807e+09,6.56345e+09,raptor,1.15
arch-a,resnet18,throughput,4,batch,PASS,FAIL,19.2072,13.4585,52.1,74.3,380.1,481.604,7.95703e+09,6.56345e+09,pimcomp,1.43
arch-a,resnet18,throughput,8,batch,PASS,FAIL,14.1578,13.4585,70.6,74.3,453.416,481.604,6.77909e+09,6.56345e+09,pimcomp,1.05
arch-a,resnet34,latency,1,element,PASS,FAIL,29.7186,66.7114,NA,NA,411.806,277.485,1.22383e+10,1.85114e+10,raptor,2.24
arch-a,resnet34,throughput,2,batch,PASS,FAIL,23.2705,23.5817,43,42.4,471.717,515.68,1.3055e+10,1.22836e+10,raptor,1.01
arch-a,resnet34,throughput,4,batch,PASS,FAIL,27.468,23.5817,36.4,42.4,422.223,515.68,1.48991e+10,1.22836e+10,pimcomp,1.16
arch-a,resnet34,throughput,8,batch,PASS,FAIL,39.8392,23.5817,25.1,42.4,352.28,515.68,1.70627e+10,1.22836e+10,pimcomp,1.69
arch-a,googlenet,latency,1,element,PASS,PASS,9.77772,62.9248,NA,NA,563.864,231.193,5.5133e+09,1.45477e+10,raptor,6.44
arch-a,googlenet,throughput,2,batch,PASS,FAIL,8.41545,18.2643,119,54.8,610.746,435.107,5.77746e+09,8.16479e+09,raptor,2.17
arch-a,googlenet,throughput,4,batch,PASS,FAIL,10.7292,18.2643,93.2,54.8,528.026,435.107,6.59289e+09,8.16479e+09,raptor,1.70
arch-a,googlenet,throughput,8,batch,PASS,FAIL,10.1219,18.2643,98.8,54.8,523.808,435.107,6.52174e+09,8.16479e+09,raptor,1.80
arch-a,yolo11n,latency,1,element,PASS,FAIL,147.49,NA,NA,NA,290.174,NA,4.27977e+10,NA,NA,NA
arch-a,yolo11n,throughput,2,batch,SKIPPED,FAIL,102.337,NA,9.77,NA,337.882,NA,3.83081e+10,NA,NA,NA
arch-a,yolo11n,throughput,4,batch,SKIPPED,FAIL,87.9103,NA,11.4,NA,358.201,NA,3.87298e+10,NA,NA,NA
arch-a,yolo11n,throughput,8,batch,SKIPPED,FAIL,82.9508,NA,12.1,NA,366.725,NA,3.85347e+10,NA,NA,NA
arch-a,vgg8,throughput,2,batch,PASS,FAIL,0.625998,1.0364,1600,965,523.278,482.597,4.17419e+08,5.41692e+08,NA,1.66
arch-a,vgg8,throughput,4,batch,PASS,FAIL,0.625998,1.0364,1600,965,523.278,482.597,4.17419e+08,5.41692e+08,NA,1.66
arch-a,vgg8,throughput,8,batch,PASS,FAIL,0.62635,1.0364,1600,965,519.182,482.597,4.14242e+08,5.41692e+08,NA,1.66
arch-a,resnet18,latency,1,element,PASS,FAIL,14.0332,58.1775,NA,NA,455.67,238.389,6.3945e+09,1.38689e+10,NA,4.15
arch-a,resnet18,throughput,2,batch,PASS,FAIL,11.0904,13.4585,90.2,74.3,520.244,481.604,6.55167e+09,6.56345e+09,NA,1.21
arch-a,resnet18,throughput,4,batch,PASS,FAIL,12.0822,13.4585,82.8,74.3,487.576,481.604,6.91662e+09,6.56345e+09,NA,1.11
arch-a,resnet18,throughput,8,batch,PASS,FAIL,11.9279,13.4585,83.8,74.3,499.354,481.604,6.84729e+09,6.56345e+09,NA,1.13
arch-a,resnet34,latency,1,element,PASS,FAIL,29.7189,66.7114,NA,NA,411.804,277.485,1.22384e+10,1.85114e+10,NA,2.24
arch-a,resnet34,throughput,2,batch,PASS,FAIL,21.8196,23.5817,45.8,42.4,477.108,515.68,1.57899e+10,1.22836e+10,NA,1.08
arch-a,resnet34,throughput,4,batch,PASS,FAIL,20,23.5817,50,42.4,548.372,515.68,1.09674e+10,1.22836e+10,NA,1.18
arch-a,resnet34,throughput,8,batch,PASS,FAIL,20.6363,23.5817,48.5,42.4,504.854,515.68,1.23725e+10,1.22836e+10,NA,1.14
arch-a,googlenet,latency,1,element,PASS,FAIL,9.77731,62.9248,NA,NA,563.88,231.193,5.51323e+09,1.45477e+10,NA,6.44
arch-a,googlenet,throughput,2,batch,PASS,FAIL,7.34567,18.2643,136,54.8,679.351,435.107,5.5053e+09,8.16479e+09,NA,2.48
arch-a,googlenet,throughput,4,batch,PASS,FAIL,7.43728,18.2643,134,54.8,674.025,435.107,5.60809e+09,8.16479e+09,NA,2.45
arch-a,googlenet,throughput,8,batch,PASS,FAIL,9.01886,18.2643,111,54.8,595.582,435.107,5.71605e+09,8.16479e+09,NA,2.03
arch-a,yolo11n,latency,1,element,PASS,FAIL,149.46,NA,NA,NA,288.579,NA,4.31311e+10,NA,NA,NA
arch-a,yolo11n,throughput,2,batch,PASS,FAIL,94.946,NA,10.5,NA,346.466,NA,3.75539e+10,NA,NA,NA
arch-a,yolo11n,throughput,4,batch,PASS,FAIL,80.1193,NA,12.5,NA,377.408,NA,3.8811e+10,NA,NA,NA
arch-a,yolo11n,throughput,8,batch,PASS,FAIL,99.5488,NA,10,NA,339.331,NA,4.27113e+10,NA,NA,NA
arch-b,vgg8,latency,1,element,PASS,PASS,1.5333,7.15212,NA,NA,292.559,173.769,4.48582e+08,1.24281e+09,raptor,4.66
arch-b,vgg8,throughput,2,batch,PASS,PASS,0.685556,1.07282,1460,932,456.044,409.47,4.03694e+08,4.39289e+08,raptor,1.57
arch-b,vgg8,throughput,4,batch,PASS,PASS,0.673192,1.07282,1490,932,448.898,409.47,4.13527e+08,4.39289e+08,raptor,1.60
arch-b,vgg8,throughput,8,batch,PASS,PASS,0.751106,1.07282,1330,932,425.742,409.47,3.94458e+08,4.39289e+08,raptor,1.43
arch-b,resnet18,latency,1,element,PASS,FAIL,15.3758,64.3818,NA,NA,400.661,201.663,6.16048e+09,1.29834e+10,raptor,4.19
arch-b,resnet18,throughput,2,batch,PASS,FAIL,12.1214,13.434,82.5,74.4,461.407,484.804,6.29532e+09,6.56966e+09,raptor,1.11
arch-b,resnet18,throughput,4,batch,PASS,FAIL,11.3601,13.434,88,74.4,479.333,484.804,6.98745e+09,6.56966e+09,raptor,1.18
arch-b,resnet18,throughput,8,batch,PASS,FAIL,13.1216,13.434,76.2,74.4,441.248,484.804,6.38271e+09,6.56966e+09,raptor,1.02
arch-b,resnet34,latency,1,element,PASS,FAIL,27.9541,107.2,NA,NA,397.206,206.476,1.11035e+10,2.21342e+10,raptor,3.83
arch-b,resnet34,throughput,2,batch,PASS,FAIL,22.282,22.8454,44.9,43.8,438.238,493.332,1.42187e+10,1.13922e+10,raptor,1.03
arch-b,resnet34,throughput,4,batch,PASS,FAIL,29.0906,22.8454,34.4,43.8,364.956,493.332,1.63285e+10,1.13922e+10,pimcomp,1.27
arch-b,resnet34,throughput,8,batch,PASS,FAIL,47.68,22.8454,21,43.8,289.821,493.332,1.63464e+10,1.13922e+10,pimcomp,2.09
arch-b,googlenet,latency,1,element,PASS,PASS,11.2212,37.9367,NA,NA,483.266,242.037,5.42282e+09,9.18206e+09,raptor,3.38
arch-b,googlenet,throughput,2,batch,PASS,FAIL,9.21209,21.3902,109,46.8,545.488,417.866,5.5934e+09,9.01473e+09,raptor,2.33
arch-b,googlenet,throughput,4,batch,PASS,FAIL,8.56194,21.3902,117,46.8,555.264,417.866,5.86312e+09,9.01473e+09,raptor,2.50
arch-b,googlenet,throughput,8,batch,PASS,FAIL,10.8652,21.3902,92,46.8,483.477,417.866,6.12683e+09,9.01473e+09,raptor,1.97
arch-b,yolo11n,latency,1,element,PASS,FAIL,168.144,NA,NA,NA,245.314,NA,4.1248e+10,NA,NA,NA
arch-b,yolo11n,throughput,2,batch,SKIPPED,FAIL,104.494,NA,9.57,NA,302.121,NA,3.58015e+10,NA,NA,NA
arch-b,yolo11n,throughput,4,batch,SKIPPED,FAIL,108.247,NA,9.24,NA,297.484,NA,3.76297e+10,NA,NA,NA
arch-b,yolo11n,throughput,8,batch,SKIPPED,FAIL,100.129,NA,9.99,NA,308.833,NA,3.71808e+10,NA,NA,NA
arch-b,vgg8,throughput,2,batch,PASS,FAIL,0.624729,1.07282,1600,932,489.899,409.47,3.93036e+08,4.39289e+08,NA,1.72
arch-b,vgg8,throughput,4,batch,PASS,FAIL,0.62331,1.07282,1600,932,490.955,409.47,3.93418e+08,4.39289e+08,NA,1.72
arch-b,vgg8,throughput,8,batch,PASS,FAIL,0.62331,1.07282,1600,932,490.955,409.47,3.93418e+08,4.39289e+08,NA,1.72
arch-b,resnet18,latency,1,element,PASS,FAIL,15.3758,64.3818,NA,NA,400.661,201.663,6.16048e+09,1.29834e+10,NA,4.19
arch-b,resnet18,throughput,2,batch,PASS,FAIL,12.9113,13.434,77.5,74.4,438.931,484.804,6.47794e+09,6.56966e+09,NA,1.04
arch-b,resnet18,throughput,4,batch,PASS,FAIL,11.3505,13.434,88.1,74.4,478.606,484.804,6.66517e+09,6.56966e+09,NA,1.18
arch-b,resnet18,throughput,8,batch,PASS,FAIL,11.3685,13.434,88,74.4,478.33,484.804,6.51513e+09,6.56966e+09,NA,1.18
arch-b,resnet34,latency,1,element,PASS,FAIL,27.9541,107.2,NA,NA,397.206,206.476,1.11035e+10,2.21342e+10,NA,3.83
arch-b,resnet34,throughput,2,batch,PASS,FAIL,22.4967,22.8454,44.5,43.8,440.06,493.332,1.49623e+10,1.13922e+10,NA,1.02
arch-b,resnet34,throughput,4,batch,PASS,FAIL,24.1838,22.8454,41.3,43.8,426.129,493.332,1.30195e+10,1.13922e+10,NA,1.06
arch-b,resnet34,throughput,8,batch,PASS,FAIL,24.1516,22.8454,41.4,43.8,420.745,493.332,1.39184e+10,1.13922e+10,NA,1.06
arch-b,googlenet,latency,1,element,PASS,FAIL,11.2206,37.9367,NA,NA,483.283,242.037,5.42274e+09,9.18206e+09,NA,3.38
arch-b,googlenet,throughput,2,batch,PASS,FAIL,9.08396,21.3902,110,46.8,550.292,417.866,5.70509e+09,9.01473e+09,NA,2.35
arch-b,googlenet,throughput,4,batch,PASS,FAIL,8.23423,21.3902,121,46.8,597.327,417.866,5.62683e+09,9.01473e+09,NA,2.59
arch-b,googlenet,throughput,8,batch,PASS,FAIL,7.78944,21.3902,128,46.8,620.65,417.866,5.47039e+09,9.01473e+09,NA,2.74
arch-b,yolo11n,latency,1,element,PASS,FAIL,171.598,NA,NA,NA,243.177,NA,4.17288e+10,NA,NA,NA
arch-b,yolo11n,throughput,2,batch,PASS,FAIL,99.5318,NA,10,NA,309.102,NA,3.5219e+10,NA,NA,NA
arch-b,yolo11n,throughput,4,batch,PASS,FAIL,85.3497,NA,11.7,NA,326.712,NA,3.58691e+10,NA,NA,NA
arch-b,yolo11n,throughput,8,batch,PASS,FAIL,97.9941,NA,10.2,NA,308.041,NA,3.5011e+10,NA,NA,NA
1 arch model mode raptor_pipeline pimcomp_pipeline raptor_functional_validation pimcomp_functional_validation raptor_latency_ms pimcomp_latency_ms raptor_throughput_samples_s pimcomp_throughput_samples_s raptor_power_mw pimcomp_power_mw raptor_energy_pj pimcomp_energy_pj better_compiler speedup
2 arch-a vgg8 latency 1 element PASS PASS 1.51833 7.98507 NA NA 321.046 200.111 4.87455e+08 1.5979e+09 raptor 5.26
3 arch-a vgg8 throughput 2 batch PASS PASS FAIL 0.906321 0.625998 1.0364 1100 1600 965 429.311 523.278 482.597 4.07972e+08 4.17419e+08 5.41692e+08 raptor NA 1.14 1.66
4 arch-a vgg8 throughput 4 batch PASS PASS FAIL 0.778242 0.625998 1.0364 1280 1600 965 457.039 523.278 482.597 4.15762e+08 4.17419e+08 5.41692e+08 raptor NA 1.33 1.66
5 arch-a vgg8 throughput 8 batch PASS PASS FAIL 0.778094 0.62635 1.0364 1290 1600 965 464.071 519.182 482.597 3.96424e+08 4.14242e+08 5.41692e+08 raptor NA 1.34 1.66
6 arch-a resnet18 latency 1 element PASS FAIL 14.0332 58.1775 NA NA 455.67 238.389 6.3945e+09 1.38689e+10 raptor NA 4.15
7 arch-a resnet18 throughput 2 batch PASS FAIL 11.7314 11.0904 13.4585 85.2 90.2 74.3 503.111 520.244 481.604 6.62807e+09 6.55167e+09 6.56345e+09 raptor NA 1.15 1.21
8 arch-a resnet18 throughput 4 batch PASS FAIL 19.2072 12.0822 13.4585 52.1 82.8 74.3 380.1 487.576 481.604 7.95703e+09 6.91662e+09 6.56345e+09 pimcomp NA 1.43 1.11
9 arch-a resnet18 throughput 8 batch PASS FAIL 14.1578 11.9279 13.4585 70.6 83.8 74.3 453.416 499.354 481.604 6.77909e+09 6.84729e+09 6.56345e+09 pimcomp NA 1.05 1.13
10 arch-a resnet34 latency 1 element PASS FAIL 29.7186 29.7189 66.7114 NA NA 411.806 411.804 277.485 1.22383e+10 1.22384e+10 1.85114e+10 raptor NA 2.24
11 arch-a resnet34 throughput 2 batch PASS FAIL 23.2705 21.8196 23.5817 43 45.8 42.4 471.717 477.108 515.68 1.3055e+10 1.57899e+10 1.22836e+10 raptor NA 1.01 1.08
12 arch-a resnet34 throughput 4 batch PASS FAIL 27.468 20 23.5817 36.4 50 42.4 422.223 548.372 515.68 1.48991e+10 1.09674e+10 1.22836e+10 pimcomp NA 1.16 1.18
13 arch-a resnet34 throughput 8 batch PASS FAIL 39.8392 20.6363 23.5817 25.1 48.5 42.4 352.28 504.854 515.68 1.70627e+10 1.23725e+10 1.22836e+10 pimcomp NA 1.69 1.14
14 arch-a googlenet latency 1 element PASS PASS FAIL 9.77772 9.77731 62.9248 NA NA 563.864 563.88 231.193 5.5133e+09 5.51323e+09 1.45477e+10 raptor NA 6.44
15 arch-a googlenet throughput 2 batch PASS FAIL 8.41545 7.34567 18.2643 119 136 54.8 610.746 679.351 435.107 5.77746e+09 5.5053e+09 8.16479e+09 raptor NA 2.17 2.48
16 arch-a googlenet throughput 4 batch PASS FAIL 10.7292 7.43728 18.2643 93.2 134 54.8 528.026 674.025 435.107 6.59289e+09 5.60809e+09 8.16479e+09 raptor NA 1.70 2.45
17 arch-a googlenet throughput 8 batch PASS FAIL 10.1219 9.01886 18.2643 98.8 111 54.8 523.808 595.582 435.107 6.52174e+09 5.71605e+09 8.16479e+09 raptor NA 1.80 2.03
18 arch-a yolo11n latency 1 element PASS FAIL 147.49 149.46 NA NA NA 290.174 288.579 NA 4.27977e+10 4.31311e+10 NA NA NA
19 arch-a yolo11n throughput 2 batch SKIPPED PASS FAIL 102.337 94.946 NA 9.77 10.5 NA 337.882 346.466 NA 3.83081e+10 3.75539e+10 NA NA NA
20 arch-a yolo11n throughput 4 batch SKIPPED PASS FAIL 87.9103 80.1193 NA 11.4 12.5 NA 358.201 377.408 NA 3.87298e+10 3.8811e+10 NA NA NA
21 arch-a yolo11n throughput 8 batch SKIPPED PASS FAIL 82.9508 99.5488 NA 12.1 10 NA 366.725 339.331 NA 3.85347e+10 4.27113e+10 NA NA NA
22 arch-b vgg8 latency 1 element PASS PASS 1.5333 7.15212 NA NA 292.559 173.769 4.48582e+08 1.24281e+09 raptor 4.66
23 arch-b vgg8 throughput 2 batch PASS PASS FAIL 0.685556 0.624729 1.07282 1460 1600 932 456.044 489.899 409.47 4.03694e+08 3.93036e+08 4.39289e+08 raptor NA 1.57 1.72
24 arch-b vgg8 throughput 4 batch PASS PASS FAIL 0.673192 0.62331 1.07282 1490 1600 932 448.898 490.955 409.47 4.13527e+08 3.93418e+08 4.39289e+08 raptor NA 1.60 1.72
25 arch-b vgg8 throughput 8 batch PASS PASS FAIL 0.751106 0.62331 1.07282 1330 1600 932 425.742 490.955 409.47 3.94458e+08 3.93418e+08 4.39289e+08 raptor NA 1.43 1.72
26 arch-b resnet18 latency 1 element PASS FAIL 15.3758 64.3818 NA NA 400.661 201.663 6.16048e+09 1.29834e+10 raptor NA 4.19
27 arch-b resnet18 throughput 2 batch PASS FAIL 12.1214 12.9113 13.434 82.5 77.5 74.4 461.407 438.931 484.804 6.29532e+09 6.47794e+09 6.56966e+09 raptor NA 1.11 1.04
28 arch-b resnet18 throughput 4 batch PASS FAIL 11.3601 11.3505 13.434 88 88.1 74.4 479.333 478.606 484.804 6.98745e+09 6.66517e+09 6.56966e+09 raptor NA 1.18
29 arch-b resnet18 throughput 8 batch PASS FAIL 13.1216 11.3685 13.434 76.2 88 74.4 441.248 478.33 484.804 6.38271e+09 6.51513e+09 6.56966e+09 raptor NA 1.02 1.18
30 arch-b resnet34 latency 1 element PASS FAIL 27.9541 107.2 NA NA 397.206 206.476 1.11035e+10 2.21342e+10 raptor NA 3.83
31 arch-b resnet34 throughput 2 batch PASS FAIL 22.282 22.4967 22.8454 44.9 44.5 43.8 438.238 440.06 493.332 1.42187e+10 1.49623e+10 1.13922e+10 raptor NA 1.03 1.02
32 arch-b resnet34 throughput 4 batch PASS FAIL 29.0906 24.1838 22.8454 34.4 41.3 43.8 364.956 426.129 493.332 1.63285e+10 1.30195e+10 1.13922e+10 pimcomp NA 1.27 1.06
33 arch-b resnet34 throughput 8 batch PASS FAIL 47.68 24.1516 22.8454 21 41.4 43.8 289.821 420.745 493.332 1.63464e+10 1.39184e+10 1.13922e+10 pimcomp NA 2.09 1.06
34 arch-b googlenet latency 1 element PASS PASS FAIL 11.2212 11.2206 37.9367 NA NA 483.266 483.283 242.037 5.42282e+09 5.42274e+09 9.18206e+09 raptor NA 3.38
35 arch-b googlenet throughput 2 batch PASS FAIL 9.21209 9.08396 21.3902 109 110 46.8 545.488 550.292 417.866 5.5934e+09 5.70509e+09 9.01473e+09 raptor NA 2.33 2.35
36 arch-b googlenet throughput 4 batch PASS FAIL 8.56194 8.23423 21.3902 117 121 46.8 555.264 597.327 417.866 5.86312e+09 5.62683e+09 9.01473e+09 raptor NA 2.50 2.59
37 arch-b googlenet throughput 8 batch PASS FAIL 10.8652 7.78944 21.3902 92 128 46.8 483.477 620.65 417.866 6.12683e+09 5.47039e+09 9.01473e+09 raptor NA 1.97 2.74
38 arch-b yolo11n latency 1 element PASS FAIL 168.144 171.598 NA NA NA 245.314 243.177 NA 4.1248e+10 4.17288e+10 NA NA NA
39 arch-b yolo11n throughput 2 batch SKIPPED PASS FAIL 104.494 99.5318 NA 9.57 10 NA 302.121 309.102 NA 3.58015e+10 3.5219e+10 NA NA NA
40 arch-b yolo11n throughput 4 batch SKIPPED PASS FAIL 108.247 85.3497 NA 9.24 11.7 NA 297.484 326.712 NA 3.76297e+10 3.58691e+10 NA NA NA
41 arch-b yolo11n throughput 8 batch SKIPPED PASS FAIL 100.129 97.9941 NA 9.99 10.2 NA 308.833 308.041 NA 3.71808e+10 3.5011e+10 NA NA NA
@@ -282,6 +282,23 @@ def comparison_passed(report: dict) -> bool:
return result.get("status") == "done" and bool(result.get("passed"))
def completed_synchronization_ablation(report: dict) -> bool:
"""A completed no-sync measurement may have intentionally invalid outputs."""
validation = report.get("raptor_validation") or {}
performance = report.get("raptor_performance") or {}
return (
report.get("pimsim_mode") == "throughput"
and "--pim-disable-synchronization" in report.get("raptor_extra_args", [])
and validation.get("status") == "done" and validation.get("passed") is False
and bool(validation.get("max_diffs"))
and not report.get("failures")
and bool(report.get("steps"))
and all(step.get("status") == "passed" for step in report["steps"])
and not performance.get("error")
and performance_values(performance)["throughput"] is not None
)
def functional_validation_status(result: dict | None) -> str:
if result is None:
return "NA"
@@ -869,6 +886,7 @@ def main() -> int:
assert results_path is not None
print_stage(results_path.name, STAGE_COLORS["Compare outputs"])
print(results_path.read_text(encoding="utf-8"), end="")
diagnostic_failures = []
for arch in args.archs:
for name in args.models:
for mode, pipeline, _ in comparisons_by_arch[arch]:
@@ -876,15 +894,19 @@ def main() -> int:
report_path = result_dir(
out_dir, name, arch, mode, pipeline, args.ablation_variant
) / "pimcomp/comparison_report.json"
if not report_path.exists() or not comparison_passed(
json.loads(report_path.read_text(encoding="utf-8")),
):
if label not in failed:
report = json.loads(report_path.read_text(encoding="utf-8")) if report_path.exists() else {}
if not comparison_passed(report):
if (label not in failed and args.ablation_variant == "no-sync"
and completed_synchronization_ablation(report)):
diagnostic_failures.append(label)
elif label not in failed:
failed.append(label)
print(Style.BRIGHT + Fore.CYAN + "Summary" + Style.RESET_ALL)
total_jobs = sum(len(args.models) * len(comparisons) for comparisons in comparisons_by_arch.values())
print(Style.BRIGHT + f"Passed: {total_jobs - len(failed)}" + Style.RESET_ALL)
print(Style.BRIGHT + f"Passed: {total_jobs - len(failed) - len(diagnostic_failures)}" + Style.RESET_ALL)
print(Style.BRIGHT + f"Failed: {len(failed)}" + Style.RESET_ALL)
if diagnostic_failures:
print("No-sync numerical failures (diagnostic only): " + ", ".join(diagnostic_failures))
print(Style.BRIGHT + f"Results: {results_path}" + Style.RESET_ALL)
if failed:
print(
@@ -35,13 +35,15 @@ The corresponding scheduling call is disabled in
Rust validation starts those buffers at zero, exposing the missing computation.
```bash
.venv/bin/python validation/tools/pim/pimcomp/correctness/prefill/run_experiment.py
.venv/bin/python validation/tools/pim/pimcomp/correctness/prefill/run_experiment.py \
/path/to/comparison /path/to/matching-model.onnx
```
The default uses GoogLeNet Arch-A throughput/pipeline2. Optional positional
arguments are comparison directory, model, work directory, and shared reference
directory; the default work directory is `correctness/prefill/` below the
comparison artifacts.
The comparison directory and its matching model are required. Optional positional
arguments are work directory and shared reference directory. The default work
directory is `correctness/prefill/` below the comparison artifacts.
Historical Softmax-free models can be recovered from Git for historical artifacts.
Current GoogLeNet retains Softmax; prefill alone does not correct omitted Softmax execution.
The experiment compares the original memory image with one prefilled using
[`batch_memory.py`](prefill/batch_memory.py). In the recorded case, 20 of 39
@@ -16,8 +16,6 @@ from batch_memory import prefill_batch_memory
SCRIPT_DIR = Path(__file__).resolve().parent
REPO_ROOT = SCRIPT_DIR.parents[5]
DEFAULT_COMPARISON_DIR = REPO_ROOT / "validation/networks/pimcomp_models/googlenet/artifacts/arch-a/throughput/pipeline2"
DEFAULT_MODEL = REPO_ROOT / "validation/networks/pimcomp_models/googlenet/googlenet-12-pimsim-nn.onnx"
DEFAULT_COMMON_DIR = REPO_ROOT / "validation/networks/pimcomp_models/googlenet/artifacts/common"
SIMULATOR_MANIFEST = REPO_ROOT / "backend-simulators/pim/pim-simulator/Cargo.toml"
@@ -78,8 +76,8 @@ def compare_outputs(
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("comparison_dir", nargs="?", type=Path, default=DEFAULT_COMPARISON_DIR)
parser.add_argument("model", nargs="?", type=Path, default=DEFAULT_MODEL)
parser.add_argument("comparison_dir", type=Path)
parser.add_argument("model", type=Path, help="ONNX model used to generate the comparison artifacts")
parser.add_argument("work_dir", nargs="?", type=Path)
parser.add_argument("common_dir", nargs="?", type=Path, default=DEFAULT_COMMON_DIR)
args = parser.parse_args()