diff --git a/validation/tools/pim/ablation/results_ablation.csv b/validation/tools/pim/ablation/results_ablation.csv index f994aab..03dd278 100644 --- a/validation/tools/pim/ablation/results_ablation.csv +++ b/validation/tools/pim/ablation/results_ablation.csv @@ -1,25 +1,25 @@ comparison,arch,model,mode,raptor_pipeline,baseline_variant,baseline_functional_validation,functional_validation,latency_gain_percent,throughput_gain_percent,power_gain_percent,energy_gain_percent plan,arch-a,googlenet,latency,1,no-plan,PASS,PASS,+99.14%,NA,-226.98%,+97.19% -plan,arch-a,googlenet,throughput,4,no-plan,PASS,PASS,NA,NA,-281.06%,NA -sync,arch-a,googlenet,throughput,4,no-sync,FAIL,PASS,+0.43%,+0.74%,+20.52%,+5.68% -plan,arch-a,resnet18,latency,1,no-plan,PASS,PASS,+96.96%,NA,-156.23%,+92.22% -plan,arch-a,resnet18,throughput,4,no-plan,PASS,PASS,+95.16%,+1965.00%,-161.37%,+86.05% -sync,arch-a,resnet18,throughput,4,no-sync,FAIL,PASS,+4.36%,+4.56%,+10.16%,+5.28% +plan,arch-a,googlenet,throughput,4,no-plan,PASS,PASS,NA,NA,-291.36%,NA +sync,arch-a,googlenet,throughput,4,no-sync,FAIL,PASS,-0.75%,-0.74%,+18.89%,+18.29% +plan,arch-a,resnet18,latency,1,no-plan,PASS,PASS,+96.96%,NA,-156.22%,+92.21% +plan,arch-a,resnet18,throughput,4,no-plan,PASS,PASS,+95.06%,+1925.00%,-163.63%,+86.98% +sync,arch-a,resnet18,throughput,4,no-sync,FAIL,PASS,+2.47%,+2.53%,+9.38%,+11.62% plan,arch-a,resnet34,latency,1,no-plan,PASS,PASS,+95.94%,NA,-130.05%,+90.65% -plan,arch-a,resnet34,throughput,4,no-plan,PASS,PASS,+98.18%,+5400.00%,-182.58%,+92.86% -sync,arch-a,resnet34,throughput,4,no-sync,FAIL,PASS,+1.91%,+1.85%,+10.27%,-22.48% +plan,arch-a,resnet34,throughput,4,no-plan,PASS,PASS,+98.04%,+5000.00%,-194.25%,+94.23% +sync,arch-a,resnet34,throughput,4,no-sync,FAIL,PASS,-5.88%,-5.56%,+6.56%,+1.06% plan,arch-a,vgg8,latency,1,no-plan,PASS,PASS,+94.75%,NA,-80.65%,+90.51% -plan,arch-a,vgg8,throughput,4,no-plan,PASS,PASS,+92.77%,+1279.31%,-168.29%,+82.56% -sync,arch-a,vgg8,throughput,4,no-sync,FAIL,PASS,+30.74%,+44.14%,+15.86%,+23.15% +plan,arch-a,vgg8,throughput,4,no-plan,PASS,PASS,+92.92%,+1315.93%,-176.02%,+80.46% +sync,arch-a,vgg8,throughput,4,no-sync,FAIL,PASS,+30.89%,+45.45%,+13.71%,+40.37% plan,arch-b,googlenet,latency,1,no-plan,PASS,PASS,+98.99%,NA,-239.10%,+96.59% -plan,arch-b,googlenet,throughput,4,no-plan,PASS,PASS,NA,NA,-332.90%,NA -sync,arch-b,googlenet,throughput,4,no-sync,FAIL,PASS,-2.84%,-2.92%,+5.06%,-11.67% -plan,arch-b,resnet18,latency,1,no-plan,PASS,PASS,+96.71%,NA,-171.17%,+91.07% -plan,arch-b,resnet18,throughput,4,no-plan,PASS,PASS,+95.45%,+2097.50%,-200.36%,+83.91% -sync,arch-b,resnet18,throughput,4,no-sync,FAIL,PASS,+11.13%,+12.40%,+7.14%,+3.77% +plan,arch-b,googlenet,throughput,4,no-plan,PASS,PASS,NA,NA,-343.71%,NA +sync,arch-b,googlenet,throughput,4,no-sync,FAIL,PASS,-3.79%,-3.65%,+4.08%,+0.45% +plan,arch-b,resnet18,latency,1,no-plan,PASS,PASS,+96.71%,NA,-171.15%,+91.07% +plan,arch-b,resnet18,throughput,4,no-plan,PASS,PASS,+95.29%,+2025.00%,-206.16%,+85.59% +sync,arch-b,resnet18,throughput,4,no-sync,FAIL,PASS,+9.41%,+10.39%,+5.25%,+14.17% plan,arch-b,resnet34,latency,1,no-plan,PASS,PASS,+96.23%,NA,-166.76%,+89.94% -plan,arch-b,resnet34,throughput,4,no-plan,PASS,PASS,+97.87%,+4600.00%,-187.39%,+91.15% -sync,arch-b,resnet34,throughput,4,no-sync,FAIL,PASS,-25.37%,-20.20%,+9.40%,-31.21% +plan,arch-b,resnet34,throughput,4,no-plan,PASS,PASS,+97.73%,+4300.00%,-197.24%,+93.24% +sync,arch-b,resnet34,throughput,4,no-sync,FAIL,PASS,-9.09%,-8.33%,+7.98%,-0.38% plan,arch-b,vgg8,latency,1,no-plan,PASS,PASS,+94.68%,NA,-98.04%,+89.47% -plan,arch-b,vgg8,throughput,4,no-plan,PASS,PASS,+93.60%,+1468.63%,-204.73%,+81.01% -sync,arch-b,vgg8,throughput,4,no-sync,FAIL,PASS,+26.66%,+36.75%,+17.29%,+19.50% +plan,arch-b,vgg8,throughput,4,no-plan,PASS,PASS,+93.72%,+1490.00%,-215.76%,+80.18% +sync,arch-b,vgg8,throughput,4,no-sync,FAIL,PASS,+24.04%,+31.40%,+13.68%,+34.43% diff --git a/validation/tools/pim/pimcomp/compare/results_comparison.csv b/validation/tools/pim/pimcomp/compare/results_comparison.csv index 8e736c9..1c21909 100644 --- a/validation/tools/pim/pimcomp/compare/results_comparison.csv +++ b/validation/tools/pim/pimcomp/compare/results_comparison.csv @@ -1,41 +1,41 @@ arch,model,mode,raptor_pipeline,pimcomp_pipeline,raptor_functional_validation,pimcomp_functional_validation,raptor_latency_ms,pimcomp_latency_ms,raptor_throughput_samples_s,pimcomp_throughput_samples_s,raptor_power_mw,pimcomp_power_mw,raptor_energy_pj,pimcomp_energy_pj,better_compiler,speedup arch-a,vgg8,latency,1,element,PASS,PASS,1.51833,7.98507,NA,NA,321.046,200.111,4.87455e+08,1.5979e+09,raptor,5.26 -arch-a,vgg8,throughput,2,batch,PASS,FAIL,0.625998,1.0364,1600,965,523.278,482.597,4.17419e+08,5.41692e+08,NA,1.66 -arch-a,vgg8,throughput,4,batch,PASS,FAIL,0.625649,1.0364,1600,965,519.08,482.597,4.15766e+08,5.41692e+08,NA,1.66 -arch-a,vgg8,throughput,8,batch,PASS,FAIL,0.803492,1.0364,1240,965,463.268,482.597,4.23149e+08,5.41692e+08,NA,1.28 -arch-a,resnet18,latency,1,element,PASS,FAIL,14.0331,58.1775,NA,NA,455.673,238.389,6.39448e+09,1.38689e+10,NA,4.15 -arch-a,resnet18,throughput,2,batch,PASS,FAIL,11.0904,13.4585,90.2,74.3,520.244,481.604,6.55167e+09,6.56345e+09,NA,1.21 -arch-a,resnet18,throughput,4,batch,PASS,FAIL,12.1059,13.4585,82.6,74.3,496.462,481.604,6.62592e+09,6.56345e+09,NA,1.11 -arch-a,resnet18,throughput,8,batch,PASS,FAIL,13.4996,13.4585,74.1,74.3,450.251,481.604,7.50626e+09,6.56345e+09,NA,1.00 -arch-a,resnet34,latency,1,element,PASS,FAIL,29.7186,66.7114,NA,NA,411.806,277.485,1.22383e+10,1.85114e+10,NA,2.24 -arch-a,resnet34,throughput,2,batch,PASS,FAIL,21.8196,23.5817,45.8,42.4,477.108,515.68,1.57899e+10,1.22836e+10,NA,1.08 -arch-a,resnet34,throughput,4,batch,PASS,FAIL,18.1654,23.5817,55,42.4,535.855,515.68,1.35442e+10,1.22836e+10,NA,1.30 -arch-a,resnet34,throughput,8,batch,PASS,FAIL,27.6219,23.5817,36.2,42.4,400.427,515.68,1.70388e+10,1.22836e+10,NA,1.17 -arch-a,googlenet,latency,1,element,PASS,FAIL,9.77731,62.9248,NA,NA,563.88,231.193,5.51323e+09,1.45477e+10,NA,6.44 -arch-a,googlenet,throughput,2,batch,PASS,FAIL,7.34567,18.2643,136,54.8,679.351,435.107,5.5053e+09,8.16479e+09,NA,2.48 -arch-a,googlenet,throughput,4,batch,PASS,FAIL,7.36311,18.2643,136,54.8,674.088,435.107,5.98681e+09,8.16479e+09,NA,2.48 -arch-a,googlenet,throughput,8,batch,PASS,FAIL,12.5019,18.2643,80,54.8,473.259,435.107,7.4517e+09,8.16479e+09,NA,1.46 +arch-a,vgg8,throughput,2,batch,PASS,FAIL,0.626959,0.979432,1600,1020,549.693,475.032,3.44635e+08,4.65262e+08,NA,1.57 +arch-a,vgg8,throughput,4,batch,PASS,FAIL,0.626566,0.979432,1600,1020,545.888,475.032,3.42035e+08,4.65262e+08,NA,1.57 +arch-a,vgg8,throughput,8,batch,PASS,FAIL,0.812348,0.979432,1230,1020,462.83,475.032,3.75979e+08,4.65262e+08,NA,1.21 +arch-a,resnet18,latency,1,element,PASS,FAIL,14.0332,58.177,NA,NA,455.67,238.39,6.3945e+09,1.38688e+10,NA,4.15 +arch-a,resnet18,throughput,2,batch,PASS,FAIL,11.236,13.5135,89,74,530.553,480.65,5.96127e+09,6.49526e+09,NA,1.20 +arch-a,resnet18,throughput,4,batch,PASS,FAIL,12.3457,13.5135,81,74,500.766,480.65,6.1823e+09,6.49526e+09,NA,1.09 +arch-a,resnet18,throughput,8,batch,PASS,FAIL,13.8889,13.5135,72,74,466.401,480.65,6.4778e+09,6.49526e+09,NA,1.03 +arch-a,resnet34,latency,1,element,PASS,FAIL,29.7186,66.7093,NA,NA,411.806,277.489,1.22383e+10,1.85111e+10,NA,2.24 +arch-a,resnet34,throughput,2,batch,PASS,FAIL,23.2558,23.8095,43,42,499.548,515.304,1.16174e+10,1.22691e+10,NA,1.02 +arch-a,resnet34,throughput,4,batch,PASS,FAIL,19.6078,23.8095,51,42,557.984,515.304,1.09409e+10,1.22691e+10,NA,1.21 +arch-a,resnet34,throughput,8,batch,PASS,FAIL,30.303,23.8095,33,42,424.785,515.304,1.28723e+10,1.22691e+10,NA,1.27 +arch-a,googlenet,latency,1,element,PASS,FAIL,9.77731,62.9227,NA,NA,563.88,231.195,5.51323e+09,1.45474e+10,NA,6.44 +arch-a,googlenet,throughput,2,batch,PASS,FAIL,7.40741,18.5185,135,54,692.813,433.763,5.13195e+09,8.03266e+09,NA,2.50 +arch-a,googlenet,throughput,4,batch,PASS,FAIL,7.46269,18.5185,134,54,692.311,433.763,5.1665e+09,8.03266e+09,NA,2.48 +arch-a,googlenet,throughput,8,batch,PASS,FAIL,12.8205,18.5185,78,54,477.421,433.763,6.12078e+09,8.03266e+09,NA,1.44 arch-a,yolo11n,latency,1,element,PASS,FAIL,149.46,NA,NA,NA,288.579,NA,4.31311e+10,NA,NA,NA -arch-a,yolo11n,throughput,2,batch,PASS,FAIL,92.6953,NA,10.8,NA,350.924,NA,3.70654e+10,NA,NA,NA -arch-a,yolo11n,throughput,4,batch,PASS,FAIL,80.1193,NA,12.5,NA,377.408,NA,3.8811e+10,NA,NA,NA -arch-a,yolo11n,throughput,8,batch,PASS,FAIL,79.1191,NA,12.6,NA,381.819,NA,3.89676e+10,NA,NA,NA +arch-a,yolo11n,throughput,2,batch,PASS,FAIL,111.111,NA,9,NA,355.474,NA,3.94971e+10,NA,NA,NA +arch-a,yolo11n,throughput,4,batch,PASS,FAIL,100,NA,10,NA,380.73,NA,3.8073e+10,NA,NA,NA +arch-a,yolo11n,throughput,8,batch,PASS,FAIL,100,NA,10,NA,385.138,NA,3.85138e+10,NA,NA,NA arch-b,vgg8,latency,1,element,PASS,PASS,1.5333,7.15212,NA,NA,292.559,173.769,4.48582e+08,1.24281e+09,raptor,4.66 -arch-b,vgg8,throughput,2,batch,PASS,FAIL,0.624729,1.07282,1600,932,489.899,409.47,3.93036e+08,4.39289e+08,NA,1.72 -arch-b,vgg8,throughput,4,batch,PASS,FAIL,0.626763,1.07282,1600,932,491.604,409.47,4.00645e+08,4.39289e+08,NA,1.72 -arch-b,vgg8,throughput,8,batch,PASS,FAIL,0.811204,1.07282,1230,932,432.125,409.47,3.87009e+08,4.39289e+08,NA,1.32 -arch-b,resnet18,latency,1,element,PASS,FAIL,15.3758,64.3818,NA,NA,400.661,201.663,6.16048e+09,1.29834e+10,NA,4.19 -arch-b,resnet18,throughput,2,batch,PASS,FAIL,11.3606,13.434,88,74.4,481.054,484.804,6.22494e+09,6.56966e+09,NA,1.18 -arch-b,resnet18,throughput,4,batch,PASS,FAIL,11.371,13.434,87.9,74.4,478.704,484.804,6.40922e+09,6.56966e+09,NA,1.18 -arch-b,resnet18,throughput,8,batch,PASS,FAIL,15.4273,13.434,64.8,74.4,379.916,484.804,7.28283e+09,6.56966e+09,NA,1.15 -arch-b,resnet34,latency,1,element,PASS,FAIL,27.9541,107.2,NA,NA,397.206,206.476,1.11035e+10,2.21342e+10,NA,3.83 -arch-b,resnet34,throughput,2,batch,PASS,FAIL,22.4967,22.8454,44.5,43.8,440.06,493.332,1.49623e+10,1.13922e+10,NA,1.02 -arch-b,resnet34,throughput,4,batch,PASS,FAIL,21.2922,22.8454,47,43.8,458.231,493.332,1.41126e+10,1.13922e+10,NA,1.07 -arch-b,resnet34,throughput,8,batch,PASS,FAIL,25.623,22.8454,39,43.8,387.339,493.332,1.54807e+10,1.13922e+10,NA,1.12 -arch-b,googlenet,latency,1,element,PASS,FAIL,11.2212,37.9367,NA,NA,483.266,242.037,5.42282e+09,9.18206e+09,NA,3.38 -arch-b,googlenet,throughput,2,batch,PASS,FAIL,7.48177,21.3902,134,46.8,637.953,417.866,5.51184e+09,9.01473e+09,NA,2.86 -arch-b,googlenet,throughput,4,batch,PASS,FAIL,7.5111,21.3902,133,46.8,635.353,417.866,5.53793e+09,9.01473e+09,NA,2.84 -arch-b,googlenet,throughput,8,batch,PASS,FAIL,9.95219,21.3902,100,46.8,492.141,417.866,6.39044e+09,9.01473e+09,NA,2.14 +arch-b,vgg8,throughput,2,batch,PASS,FAIL,0.627746,1.06383,1590,940,516.239,408.479,3.24067e+08,4.34552e+08,NA,1.69 +arch-b,vgg8,throughput,4,batch,PASS,FAIL,0.627746,1.06383,1590,940,519.279,408.479,3.25976e+08,4.34552e+08,NA,1.69 +arch-b,vgg8,throughput,8,batch,PASS,FAIL,0.813008,1.06383,1230,940,432.62,408.479,3.51723e+08,4.34552e+08,NA,1.31 +arch-b,resnet18,latency,1,element,PASS,FAIL,15.3758,64.3808,NA,NA,400.661,201.664,6.16048e+09,1.29833e+10,NA,4.19 +arch-b,resnet18,throughput,2,batch,PASS,FAIL,11.4943,13.5135,87,74,491.763,483.916,5.65245e+09,6.53941e+09,NA,1.18 +arch-b,resnet18,throughput,4,batch,PASS,FAIL,11.7647,13.5135,85,74,487.946,483.916,5.74054e+09,6.53941e+09,NA,1.15 +arch-b,resnet18,throughput,8,batch,PASS,FAIL,16.129,13.5135,62,74,396.088,483.916,6.38852e+09,6.53941e+09,NA,1.19 +arch-b,resnet34,latency,1,element,PASS,FAIL,27.9541,107.201,NA,NA,397.206,206.475,1.11035e+10,2.21343e+10,NA,3.83 +arch-b,resnet34,throughput,2,batch,PASS,FAIL,24.3902,23.2558,41,43,454.795,492.806,1.10926e+10,1.14606e+10,NA,1.05 +arch-b,resnet34,throughput,4,batch,PASS,FAIL,22.7273,23.2558,44,43,473.936,492.806,1.07713e+10,1.14606e+10,NA,1.02 +arch-b,resnet34,throughput,8,batch,PASS,FAIL,28.5714,23.2558,35,43,414.549,492.806,1.18443e+10,1.14606e+10,NA,1.23 +arch-b,googlenet,latency,1,element,PASS,FAIL,11.2209,37.9362,NA,NA,483.274,242.038,5.42278e+09,9.182e+09,NA,3.38 +arch-b,googlenet,throughput,2,batch,PASS,FAIL,7.57576,21.7391,132,46,653.673,417.304,4.95207e+09,9.07182e+09,NA,2.87 +arch-b,googlenet,throughput,4,batch,PASS,FAIL,7.57576,21.7391,132,46,651.226,417.304,4.93353e+09,9.07182e+09,NA,2.87 +arch-b,googlenet,throughput,8,batch,PASS,FAIL,10.2041,21.7391,98,46,522.741,417.304,5.3341e+09,9.07182e+09,NA,2.13 arch-b,yolo11n,latency,1,element,PASS,FAIL,171.598,NA,NA,NA,243.177,NA,4.17288e+10,NA,NA,NA -arch-b,yolo11n,throughput,2,batch,PASS,FAIL,97.269,NA,10.3,NA,311.245,NA,3.46934e+10,NA,NA,NA -arch-b,yolo11n,throughput,4,batch,PASS,FAIL,78.8274,NA,12.7,NA,340.026,NA,3.39053e+10,NA,NA,NA -arch-b,yolo11n,throughput,8,batch,PASS,FAIL,86.2414,NA,11.6,NA,323.447,NA,3.7648e+10,NA,NA,NA +arch-b,yolo11n,throughput,2,batch,PASS,FAIL,111.111,NA,9,NA,315.563,NA,3.50626e+10,NA,NA,NA +arch-b,yolo11n,throughput,4,batch,PASS,FAIL,90.9091,NA,11,NA,349.244,NA,3.17494e+10,NA,NA,NA +arch-b,yolo11n,throughput,8,batch,PASS,FAIL,111.111,NA,9,NA,332.438,NA,3.69376e+10,NA,NA,NA