This commit is contained in:
@@ -0,0 +1,178 @@
|
|||||||
|
Operation,Arch,Result (l),Result (t),Compile (l),Host mem (l),Cores mem (l),Cores (l),Xbars (l),Latency (l),Power (l),Energy (l),Compile (t),Host mem (t),Cores mem (t),Cores (t),Xbars (t),Avg latency (t),Throughput (t),Avg power (t),Avg energy (t)
|
||||||
|
add/after_gemm,arch-a,PASS,PASS,0.040 s,0.01 MiB,0.01 MiB,4,4,SKIP,SKIP,SKIP,0.041 s,0.01 MiB,0.01 MiB,5,4,SKIP,SKIP,SKIP,SKIP
|
||||||
|
add/basic,arch-a,PASS,PASS,0.033 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.034 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
add/broadcast_row,arch-a,PASS,PASS,0.034 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.034 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
add/channel_broadcast_1024,arch-a,PASS,PASS,0.035 s,0.02 MiB,0.01 MiB,1,0,SKIP,SKIP,SKIP,0.035 s,0.02 MiB,0.01 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
add/leading_dimension_broadcast,arch-a,PASS,PASS,0.035 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.035 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
concat/channel_axis,arch-a,PASS,PASS,0.035 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.035 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
concat/negative_axis,arch-a,PASS,PASS,0.033 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.035 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
concat/three_inputs_channel_axis,arch-a,PASS,PASS,0.036 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.035 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
conv/batch_2,arch-a,PASS,PASS,0.042 s,0.00 MiB,0.00 MiB,2,2,SKIP,SKIP,SKIP,0.050 s,0.00 MiB,0.01 MiB,4,2,SKIP,SKIP,SKIP,SKIP
|
||||||
|
conv/batch_4_pointwise,arch-a,PASS,PASS,0.044 s,0.00 MiB,0.01 MiB,4,4,SKIP,SKIP,SKIP,0.052 s,0.01 MiB,0.01 MiB,5,4,SKIP,SKIP,SKIP,SKIP
|
||||||
|
conv/depthwise_1024_channels,arch-a,PASS,PASS,0.068 s,0.19 MiB,0.38 MiB,128,128,SKIP,SKIP,SKIP,0.116 s,0.26 MiB,0.50 MiB,130,128,SKIP,SKIP,SKIP,SKIP
|
||||||
|
conv/depthwise_grouped,arch-a,PASS,PASS,0.039 s,0.01 MiB,0.00 MiB,4,4,SKIP,SKIP,SKIP,0.044 s,0.01 MiB,0.00 MiB,6,4,SKIP,SKIP,SKIP,SKIP
|
||||||
|
conv/dilated_3x3,arch-a,PASS,PASS,0.050 s,0.01 MiB,0.01 MiB,9,9,SKIP,SKIP,SKIP,0.050 s,0.01 MiB,0.01 MiB,10,9,SKIP,SKIP,SKIP,SKIP
|
||||||
|
conv/dynamic,arch-a,PASS,PASS,0.039 s,0.00 MiB,0.00 MiB,5,0,SKIP,SKIP,SKIP,0.045 s,0.00 MiB,0.00 MiB,5,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
conv/explicit_padding,arch-a,PASS,PASS,0.042 s,0.01 MiB,0.02 MiB,16,16,SKIP,SKIP,SKIP,0.055 s,0.01 MiB,0.02 MiB,17,16,SKIP,SKIP,SKIP,SKIP
|
||||||
|
conv/grouped_many_groups,arch-a,PASS,PASS,0.547 s,0.05 MiB,0.09 MiB,64,64,SKIP,SKIP,SKIP,0.628 s,0.05 MiB,0.14 MiB,124,64,SKIP,SKIP,SKIP,SKIP
|
||||||
|
conv/grouped_two_groups,arch-a,PASS,PASS,0.049 s,0.00 MiB,0.00 MiB,2,2,SKIP,SKIP,SKIP,0.053 s,0.00 MiB,0.00 MiB,4,2,SKIP,SKIP,SKIP,SKIP
|
||||||
|
conv/huge_pointwise_1024,arch-a,PASS,PASS,0.160 s,0.01 MiB,0.09 MiB,64,64,SKIP,SKIP,SKIP,0.190 s,0.02 MiB,0.11 MiB,74,64,SKIP,SKIP,SKIP,SKIP
|
||||||
|
conv/huge_pointwise_1024_dynamic,arch-a,PASS,PASS,0.073 s,8.04 MiB,12.55 MiB,167,0,SKIP,SKIP,SKIP,0.249 s,8.05 MiB,12.45 MiB,165,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
conv/input_224_7x7_stride2,arch-a,PASS,PASS,0.777 s,24.14 MiB,61.87 MiB,168,169,SKIP,SKIP,SKIP,3.669 s,31.18 MiB,68.93 MiB,165,169,SKIP,SKIP,SKIP,SKIP
|
||||||
|
conv/kernel_2x2,arch-a,PASS,PASS,0.041 s,0.00 MiB,0.00 MiB,1,1,SKIP,SKIP,SKIP,0.063 s,0.00 MiB,0.00 MiB,3,1,SKIP,SKIP,SKIP,SKIP
|
||||||
|
conv/kernel_3x3,arch-a,PASS,PASS,0.043 s,0.01 MiB,0.01 MiB,9,9,SKIP,SKIP,SKIP,0.046 s,0.01 MiB,0.01 MiB,11,9,SKIP,SKIP,SKIP,SKIP
|
||||||
|
conv/kernel_equals_input_spatial,arch-a,PASS,PASS,0.042 s,0.00 MiB,0.00 MiB,2,2,SKIP,SKIP,SKIP,0.048 s,0.00 MiB,0.00 MiB,4,2,SKIP,SKIP,SKIP,SKIP
|
||||||
|
conv/large_input_channels_1x1,arch-a,PASS,PASS,0.079 s,0.01 MiB,0.01 MiB,8,8,SKIP,SKIP,SKIP,0.081 s,0.01 MiB,0.02 MiB,10,8,SKIP,SKIP,SKIP,SKIP
|
||||||
|
conv/large_output_channels_1x1,arch-a,PASS,PASS,0.081 s,0.00 MiB,0.01 MiB,8,8,SKIP,SKIP,SKIP,0.087 s,0.01 MiB,0.02 MiB,9,8,SKIP,SKIP,SKIP,SKIP
|
||||||
|
conv/large_spatial,arch-a,PASS,PASS,0.048 s,0.01 MiB,0.04 MiB,36,36,SKIP,SKIP,SKIP,0.069 s,0.02 MiB,0.04 MiB,37,36,SKIP,SKIP,SKIP,SKIP
|
||||||
|
conv/multi_channel,arch-a,PASS,PASS,0.043 s,0.00 MiB,0.00 MiB,3,3,SKIP,SKIP,SKIP,0.046 s,0.00 MiB,0.00 MiB,4,3,SKIP,SKIP,SKIP,SKIP
|
||||||
|
conv/non_square_kernel_1x3,arch-a,PASS,PASS,0.045 s,0.00 MiB,0.00 MiB,2,2,SKIP,SKIP,SKIP,0.047 s,0.00 MiB,0.00 MiB,3,2,SKIP,SKIP,SKIP,SKIP
|
||||||
|
conv/non_square_kernel_3x1,arch-a,PASS,PASS,0.042 s,0.00 MiB,0.00 MiB,2,2,SKIP,SKIP,SKIP,0.046 s,0.00 MiB,0.00 MiB,3,2,SKIP,SKIP,SKIP,SKIP
|
||||||
|
conv/non_uniform_stride,arch-a,PASS,PASS,0.045 s,0.00 MiB,0.00 MiB,3,3,SKIP,SKIP,SKIP,0.046 s,0.00 MiB,0.00 MiB,4,3,SKIP,SKIP,SKIP,SKIP
|
||||||
|
conv/output_channel_grouping_minimal,arch-a,PASS,PASS,0.065 s,0.10 MiB,0.28 MiB,128,128,SKIP,SKIP,SKIP,0.104 s,0.10 MiB,0.25 MiB,128,128,SKIP,SKIP,SKIP,SKIP
|
||||||
|
conv/pointwise_1x1,arch-a,PASS,PASS,0.044 s,0.00 MiB,0.00 MiB,1,1,SKIP,SKIP,SKIP,0.044 s,0.00 MiB,0.00 MiB,2,1,SKIP,SKIP,SKIP,SKIP
|
||||||
|
conv/pointwise_tiled_chain,arch-a,PASS,PASS,0.738 s,0.01 MiB,0.03 MiB,17,80,SKIP,SKIP,SKIP,0.745 s,0.02 MiB,0.08 MiB,22,80,SKIP,SKIP,SKIP,SKIP
|
||||||
|
conv/real_asymmetric_padding,arch-a,PASS,PASS,0.044 s,0.01 MiB,0.03 MiB,28,28,SKIP,SKIP,SKIP,0.060 s,0.01 MiB,0.03 MiB,29,28,SKIP,SKIP,SKIP,SKIP
|
||||||
|
conv/relu_conv_store,arch-a,PASS,PASS,0.070 s,0.16 MiB,0.66 MiB,168,168,SKIP,SKIP,SKIP,0.120 s,0.17 MiB,0.66 MiB,166,166,SKIP,SKIP,SKIP,SKIP
|
||||||
|
conv/same_lower_3x3,arch-a,PASS,PASS,0.043 s,0.01 MiB,0.02 MiB,25,25,SKIP,SKIP,SKIP,0.053 s,0.01 MiB,0.02 MiB,26,25,SKIP,SKIP,SKIP,SKIP
|
||||||
|
conv/same_padding_3x3,arch-a,PASS,PASS,0.046 s,0.01 MiB,0.02 MiB,25,25,SKIP,SKIP,SKIP,0.056 s,0.01 MiB,0.02 MiB,26,25,SKIP,SKIP,SKIP,SKIP
|
||||||
|
conv/strategy_depthwise_16,arch-a,PASS,PASS,0.062 s,0.06 MiB,0.33 MiB,168,168,SKIP,SKIP,SKIP,0.125 s,0.08 MiB,0.11 MiB,27,52,SKIP,SKIP,SKIP,SKIP
|
||||||
|
conv/strategy_input_k_tiled,arch-a,PASS,PASS,0.061 s,0.08 MiB,0.27 MiB,108,108,SKIP,SKIP,SKIP,0.091 s,0.08 MiB,0.32 MiB,110,108,SKIP,SKIP,SKIP,SKIP
|
||||||
|
conv/strategy_output_channel_tiled,arch-a,PASS,PASS,0.062 s,0.03 MiB,0.15 MiB,72,72,SKIP,SKIP,SKIP,0.078 s,0.04 MiB,0.13 MiB,73,72,SKIP,SKIP,SKIP,SKIP
|
||||||
|
conv/strategy_streamed_packed,arch-a,PASS,PASS,0.131 s,3.34 MiB,7.89 MiB,168,168,SKIP,SKIP,SKIP,0.418 s,3.35 MiB,7.89 MiB,165,165,SKIP,SKIP,SKIP,SKIP
|
||||||
|
conv/strategy_streamed_patch,arch-a,PASS,PASS,0.074 s,0.34 MiB,1.32 MiB,168,168,SKIP,SKIP,SKIP,0.159 s,0.35 MiB,1.32 MiB,165,165,SKIP,SKIP,SKIP,SKIP
|
||||||
|
conv/strategy_tiled_2d,arch-a,PASS,PASS,0.096 s,0.10 MiB,0.37 MiB,168,168,SKIP,SKIP,SKIP,0.179 s,0.11 MiB,0.54 MiB,165,168,SKIP,SKIP,SKIP,SKIP
|
||||||
|
conv/stride_2,arch-a,PASS,PASS,0.053 s,0.01 MiB,0.00 MiB,4,4,SKIP,SKIP,SKIP,0.053 s,0.01 MiB,0.00 MiB,6,4,SKIP,SKIP,SKIP,SKIP
|
||||||
|
conv/with_bias_3x3,arch-a,PASS,PASS,0.045 s,0.00 MiB,0.01 MiB,3,3,SKIP,SKIP,SKIP,0.047 s,0.00 MiB,0.01 MiB,4,3,SKIP,SKIP,SKIP,SKIP
|
||||||
|
conv/with_constant,arch-a,PASS,PASS,0.057 s,0.00 MiB,0.00 MiB,1,1,SKIP,SKIP,SKIP,0.069 s,0.00 MiB,0.00 MiB,3,1,SKIP,SKIP,SKIP,SKIP
|
||||||
|
conv/without_kernel_shape_attr,arch-a,PASS,PASS,0.051 s,0.01 MiB,0.01 MiB,9,9,SKIP,SKIP,SKIP,0.057 s,0.01 MiB,0.01 MiB,11,9,SKIP,SKIP,SKIP,SKIP
|
||||||
|
conv/yolo11n_depthwise_head,arch-a,PASS,PASS,2.129 s,8.66 MiB,33.48 MiB,164,250,SKIP,SKIP,SKIP,9.599 s,10.25 MiB,36.90 MiB,166,255,SKIP,SKIP,SKIP,SKIP
|
||||||
|
conv/yolo11n_heavy,arch-a,PASS,PASS,0.188 s,4.82 MiB,19.00 MiB,160,800,SKIP,SKIP,SKIP,0.298 s,4.83 MiB,19.10 MiB,161,800,SKIP,SKIP,SKIP,SKIP
|
||||||
|
conv/yolo11n_stem,arch-a,PASS,PASS,0.873 s,12.85 MiB,31.85 MiB,168,488,SKIP,SKIP,SKIP,1.318 s,12.88 MiB,31.68 MiB,165,485,SKIP,SKIP,SKIP,SKIP
|
||||||
|
div/after_gemm,arch-a,PASS,PASS,0.046 s,0.01 MiB,0.01 MiB,4,4,SKIP,SKIP,SKIP,0.052 s,0.01 MiB,0.01 MiB,5,4,SKIP,SKIP,SKIP,SKIP
|
||||||
|
div/basic,arch-a,PASS,PASS,0.042 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.044 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
div/channel_broadcast_1024,arch-a,PASS,PASS,0.048 s,0.02 MiB,0.01 MiB,1,0,SKIP,SKIP,SKIP,0.040 s,0.02 MiB,0.01 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
div/leading_dimension_broadcast,arch-a,PASS,PASS,0.053 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.058 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
div/runtime_scalar_rhs,arch-a,PASS,PASS,0.043 s,0.02 MiB,0.01 MiB,1,0,SKIP,SKIP,SKIP,0.050 s,0.02 MiB,0.01 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
div/scalar_constant,arch-a,PASS,PASS,0.047 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.043 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
gather/3d_input_axis1,arch-a,PASS,PASS,0.041 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.045 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
gather/axis0_matrix_indices,arch-a,PASS,PASS,0.040 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.039 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
gather/axis1,arch-a,PASS,PASS,0.040 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.040 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
gather/negative_axis,arch-a,PASS,PASS,0.041 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.042 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
gather/negative_indices,arch-a,PASS,PASS,0.061 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.049 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
gemm/alpha_beta,arch-a,PASS,PASS,0.050 s,0.01 MiB,0.01 MiB,4,4,SKIP,SKIP,SKIP,0.048 s,0.01 MiB,0.01 MiB,5,4,SKIP,SKIP,SKIP,SKIP
|
||||||
|
gemm/bias_rank2_broadcast,arch-a,PASS,PASS,0.045 s,0.00 MiB,0.01 MiB,4,4,SKIP,SKIP,SKIP,0.062 s,0.01 MiB,0.01 MiB,5,4,SKIP,SKIP,SKIP,SKIP
|
||||||
|
gemm/dynamic,arch-a,PASS,PASS,0.043 s,0.00 MiB,0.00 MiB,4,0,SKIP,SKIP,SKIP,0.049 s,0.00 MiB,0.00 MiB,5,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
gemm/dynamic_alpha,arch-a,PASS,PASS,0.045 s,0.00 MiB,0.00 MiB,4,0,SKIP,SKIP,SKIP,0.045 s,0.00 MiB,0.00 MiB,5,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
gemm/dynamic_beta,arch-a,PASS,PASS,0.057 s,0.00 MiB,0.00 MiB,4,0,SKIP,SKIP,SKIP,0.058 s,0.00 MiB,0.00 MiB,5,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
gemm/dynamic_bias,arch-a,PASS,PASS,0.047 s,0.00 MiB,0.00 MiB,4,0,SKIP,SKIP,SKIP,0.053 s,0.00 MiB,0.00 MiB,5,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
gemm/dynamic_bias_alpha_beta,arch-a,PASS,PASS,0.043 s,0.00 MiB,0.00 MiB,4,0,SKIP,SKIP,SKIP,0.046 s,0.00 MiB,0.00 MiB,5,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
gemm/dynamic_transpose_b,arch-a,PASS,PASS,0.076 s,0.00 MiB,0.00 MiB,4,0,SKIP,SKIP,SKIP,0.077 s,0.00 MiB,0.00 MiB,5,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
gemm/huge_1024,arch-a,PASS,PASS,0.186 s,0.01 MiB,0.09 MiB,64,64,SKIP,SKIP,SKIP,0.190 s,0.02 MiB,0.10 MiB,73,64,SKIP,SKIP,SKIP,SKIP
|
||||||
|
gemm/large,arch-a,PASS,PASS,0.050 s,0.02 MiB,0.03 MiB,16,16,SKIP,SKIP,SKIP,0.048 s,0.02 MiB,0.03 MiB,17,16,SKIP,SKIP,SKIP,SKIP
|
||||||
|
gemm/large_k_small_n,arch-a,PASS,PASS,0.113 s,0.01 MiB,0.01 MiB,8,8,SKIP,SKIP,SKIP,0.088 s,0.01 MiB,0.01 MiB,9,8,SKIP,SKIP,SKIP,SKIP
|
||||||
|
gemm/non_square,arch-a,PASS,PASS,0.046 s,0.00 MiB,0.01 MiB,4,4,SKIP,SKIP,SKIP,0.049 s,0.01 MiB,0.01 MiB,5,4,SKIP,SKIP,SKIP,SKIP
|
||||||
|
gemm/scalar_bias,arch-a,PASS,PASS,0.050 s,0.00 MiB,0.01 MiB,4,4,SKIP,SKIP,SKIP,0.057 s,0.01 MiB,0.01 MiB,5,4,SKIP,SKIP,SKIP,SKIP
|
||||||
|
gemm/small,arch-a,PASS,PASS,0.047 s,0.00 MiB,0.00 MiB,2,2,SKIP,SKIP,SKIP,0.079 s,0.00 MiB,0.00 MiB,4,2,SKIP,SKIP,SKIP,SKIP
|
||||||
|
gemm/small_k_large_n,arch-a,PASS,PASS,0.113 s,0.00 MiB,0.01 MiB,8,8,SKIP,SKIP,SKIP,0.110 s,0.01 MiB,0.02 MiB,9,8,SKIP,SKIP,SKIP,SKIP
|
||||||
|
gemm/square_weights,arch-a,PASS,PASS,0.073 s,0.03 MiB,0.08 MiB,40,40,SKIP,SKIP,SKIP,0.084 s,0.03 MiB,0.09 MiB,43,40,SKIP,SKIP,SKIP,SKIP
|
||||||
|
gemm/transpose_a,arch-a,PASS,PASS,0.055 s,0.00 MiB,0.01 MiB,4,4,SKIP,SKIP,SKIP,0.052 s,0.00 MiB,0.01 MiB,6,4,SKIP,SKIP,SKIP,SKIP
|
||||||
|
gemm/transpose_a_and_b,arch-a,PASS,PASS,0.048 s,0.00 MiB,0.01 MiB,4,4,SKIP,SKIP,SKIP,0.053 s,0.00 MiB,0.01 MiB,6,4,SKIP,SKIP,SKIP,SKIP
|
||||||
|
gemm/transpose_b,arch-a,PASS,PASS,0.065 s,0.00 MiB,0.01 MiB,4,4,SKIP,SKIP,SKIP,0.051 s,0.01 MiB,0.01 MiB,5,4,SKIP,SKIP,SKIP,SKIP
|
||||||
|
gemm/transpose_b_with_bias,arch-a,PASS,PASS,0.053 s,0.01 MiB,0.01 MiB,4,4,SKIP,SKIP,SKIP,0.060 s,0.01 MiB,0.01 MiB,5,4,SKIP,SKIP,SKIP,SKIP
|
||||||
|
gemm/with_bias,arch-a,PASS,PASS,0.054 s,0.01 MiB,0.01 MiB,4,4,SKIP,SKIP,SKIP,0.044 s,0.01 MiB,0.01 MiB,5,4,SKIP,SKIP,SKIP,SKIP
|
||||||
|
gemv/all_constant,arch-a,PASS,PASS,0.047 s,0.00 MiB,0.00 MiB,0,0,SKIP,SKIP,SKIP,0.040 s,0.00 MiB,0.00 MiB,0,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
gemv/constant_weight,arch-a,PASS,PASS,0.058 s,0.00 MiB,0.01 MiB,5,4,SKIP,SKIP,SKIP,0.059 s,0.00 MiB,0.01 MiB,7,4,SKIP,SKIP,SKIP,SKIP
|
||||||
|
gemv/non_uniform_bias,arch-a,PASS,PASS,0.062 s,0.00 MiB,0.01 MiB,5,4,SKIP,SKIP,SKIP,0.061 s,0.00 MiB,0.01 MiB,7,4,SKIP,SKIP,SKIP,SKIP
|
||||||
|
gemv/scalar_bias,arch-a,PASS,PASS,0.061 s,0.00 MiB,0.01 MiB,5,4,SKIP,SKIP,SKIP,0.072 s,0.00 MiB,0.01 MiB,7,4,SKIP,SKIP,SKIP,SKIP
|
||||||
|
gemv/uniform_bias,arch-a,PASS,PASS,0.076 s,0.00 MiB,0.01 MiB,5,4,SKIP,SKIP,SKIP,0.064 s,0.00 MiB,0.01 MiB,7,4,SKIP,SKIP,SKIP,SKIP
|
||||||
|
matmul/basic,arch-a,PASS,PASS,0.071 s,0.00 MiB,0.00 MiB,2,2,SKIP,SKIP,SKIP,0.072 s,0.00 MiB,0.00 MiB,4,2,SKIP,SKIP,SKIP,SKIP
|
||||||
|
matmul/batched_3d,arch-a,PASS,PASS,0.066 s,0.00 MiB,0.01 MiB,4,4,SKIP,SKIP,SKIP,0.060 s,0.00 MiB,0.01 MiB,5,4,SKIP,SKIP,SKIP,SKIP
|
||||||
|
matmul/batched_3d_dynamic,arch-a,PASS,PASS,0.048 s,0.00 MiB,0.00 MiB,4,0,SKIP,SKIP,SKIP,0.042 s,0.00 MiB,0.00 MiB,4,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
matmul/batched_left_constant,arch-a,PASS,PASS,0.048 s,0.00 MiB,0.02 MiB,8,8,SKIP,SKIP,SKIP,0.054 s,0.01 MiB,0.02 MiB,9,8,SKIP,SKIP,SKIP,SKIP
|
||||||
|
matmul/batched_lhs_broadcast,arch-a,PASS,PASS,0.047 s,0.00 MiB,0.01 MiB,4,4,SKIP,SKIP,SKIP,0.056 s,0.00 MiB,0.01 MiB,5,4,SKIP,SKIP,SKIP,SKIP
|
||||||
|
matmul/batched_rhs_broadcast,arch-a,PASS,PASS,0.052 s,0.00 MiB,0.01 MiB,4,4,SKIP,SKIP,SKIP,0.053 s,0.00 MiB,0.01 MiB,5,4,SKIP,SKIP,SKIP,SKIP
|
||||||
|
matmul/dynamic,arch-a,PASS,PASS,0.044 s,0.00 MiB,0.00 MiB,4,0,SKIP,SKIP,SKIP,0.054 s,0.00 MiB,0.00 MiB,5,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
matmul/huge_1024,arch-a,PASS,PASS,0.174 s,0.01 MiB,0.09 MiB,64,64,SKIP,SKIP,SKIP,0.188 s,0.02 MiB,0.10 MiB,73,64,SKIP,SKIP,SKIP,SKIP
|
||||||
|
matmul/left_constant,arch-a,PASS,PASS,0.051 s,0.00 MiB,0.01 MiB,4,4,SKIP,SKIP,SKIP,0.058 s,0.00 MiB,0.01 MiB,6,4,SKIP,SKIP,SKIP,SKIP
|
||||||
|
matmul/matrix_vector,arch-a,PASS,PASS,0.120 s,0.52 MiB,0.77 MiB,165,170,SKIP,SKIP,SKIP,0.239 s,0.53 MiB,0.77 MiB,165,173,SKIP,SKIP,SKIP,SKIP
|
||||||
|
matmul/vector_matrix,arch-a,PASS,PASS,0.088 s,0.01 MiB,0.01 MiB,8,8,SKIP,SKIP,SKIP,0.087 s,0.01 MiB,0.01 MiB,9,8,SKIP,SKIP,SKIP,SKIP
|
||||||
|
matmul/yolo_attention,arch-a,PASS,PASS,0.394 s,1.02 MiB,39.22 MiB,168,0,SKIP,SKIP,SKIP,0.512 s,2.65 MiB,26.70 MiB,43,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
mul/after_conv,arch-a,PASS,PASS,0.065 s,0.00 MiB,0.00 MiB,3,3,SKIP,SKIP,SKIP,0.081 s,0.00 MiB,0.00 MiB,4,3,SKIP,SKIP,SKIP,SKIP
|
||||||
|
mul/after_conv_scalar_constant,arch-a,PASS,PASS,0.053 s,0.00 MiB,0.00 MiB,3,3,SKIP,SKIP,SKIP,0.055 s,0.00 MiB,0.00 MiB,4,3,SKIP,SKIP,SKIP,SKIP
|
||||||
|
mul/basic,arch-a,PASS,PASS,0.075 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.079 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
mul/channel_broadcast_1024,arch-a,PASS,PASS,0.038 s,0.02 MiB,0.01 MiB,1,0,SKIP,SKIP,SKIP,0.039 s,0.02 MiB,0.01 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
mul/leading_dimension_broadcast,arch-a,PASS,PASS,0.038 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.037 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
mul/scalar_constant,arch-a,PASS,PASS,0.038 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.036 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
pool/avg_basic,arch-a,PASS,PASS,0.047 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.045 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
pool/avg_ceil_mode,arch-a,PASS,PASS,0.061 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.056 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
pool/avg_explicit_padding,arch-a,PASS,PASS,0.047 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.051 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
pool/avg_include_pad,arch-a,PASS,PASS,0.049 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.050 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
pool/avg_large_channels,arch-a,PASS,PASS,0.056 s,0.04 MiB,0.02 MiB,1,0,SKIP,SKIP,SKIP,0.059 s,0.04 MiB,0.02 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
pool/avg_non_uniform_stride,arch-a,PASS,PASS,0.050 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.053 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
pool/avg_real_asymmetric_padding,arch-a,PASS,PASS,0.048 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.048 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
pool/max_after_conv,arch-a,PASS,PASS,0.052 s,0.00 MiB,0.00 MiB,4,4,SKIP,SKIP,SKIP,0.054 s,0.00 MiB,0.00 MiB,5,4,SKIP,SKIP,SKIP,SKIP
|
||||||
|
pool/max_basic,arch-a,PASS,PASS,0.040 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.039 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
pool/max_ceil_mode,arch-a,PASS,PASS,0.042 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.044 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
pool/max_global_style_kernel_equals_input,arch-a,PASS,PASS,0.050 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.041 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
pool/max_non_square_kernel,arch-a,PASS,PASS,0.044 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.067 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
pool/max_real_asymmetric_padding,arch-a,PASS,PASS,0.059 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.059 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
pool/max_same_upper,arch-a,PASS,PASS,0.059 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.049 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
pool/max_stride2_multichannel,arch-a,PASS,PASS,0.045 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.062 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
reduce_mean/4d_spatial,arch-a,PASS,PASS,0.044 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.046 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
reduce_mean/4d_spatial_keepdims_0,arch-a,PASS,PASS,0.044 s,0.00 MiB,0.00 MiB,3,0,SKIP,SKIP,SKIP,0.045 s,0.00 MiB,0.00 MiB,4,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
reduce_mean/after_conv,arch-a,PASS,PASS,0.069 s,0.00 MiB,0.00 MiB,3,3,SKIP,SKIP,SKIP,0.067 s,0.00 MiB,0.00 MiB,4,3,SKIP,SKIP,SKIP,SKIP
|
||||||
|
reduce_mean/all_axes_keepdims_0,arch-a,PASS,PASS,0.044 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.048 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
reduce_mean/all_axes_keepdims_1,arch-a,PASS,PASS,0.041 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.052 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
reduce_mean/basic,arch-a,PASS,PASS,0.048 s,0.00 MiB,0.00 MiB,4,0,SKIP,SKIP,SKIP,0.042 s,0.00 MiB,0.00 MiB,4,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
reduce_mean/channel_axis_nchw,arch-a,PASS,PASS,0.053 s,0.03 MiB,0.02 MiB,4,0,SKIP,SKIP,SKIP,0.048 s,0.03 MiB,0.02 MiB,4,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
reduce_mean/keepdims_0,arch-a,PASS,PASS,0.046 s,0.00 MiB,0.00 MiB,4,0,SKIP,SKIP,SKIP,0.041 s,0.00 MiB,0.00 MiB,5,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
reduce_mean/large_dimension_1024,arch-a,PASS,PASS,0.037 s,0.01 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.039 s,0.01 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
reduce_mean/legacy_axes_1_2_keepdims_1,arch-a,PASS,PASS,0.041 s,0.00 MiB,0.00 MiB,2,0,SKIP,SKIP,SKIP,0.051 s,0.00 MiB,0.00 MiB,2,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
reduce_mean/legacy_axis1_keepdims_0,arch-a,PASS,PASS,0.049 s,0.00 MiB,0.00 MiB,8,0,SKIP,SKIP,SKIP,0.043 s,0.00 MiB,0.00 MiB,9,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
reduce_mean/legacy_axis1_keepdims_1,arch-a,PASS,PASS,0.043 s,0.00 MiB,0.00 MiB,8,0,SKIP,SKIP,SKIP,0.041 s,0.00 MiB,0.00 MiB,8,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
reduce_mean/legacy_empty_axes_noop,arch-a,PASS,PASS,0.035 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.040 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
reduce_mean/legacy_nchw_spatial,arch-a,PASS,PASS,0.048 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.055 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
reduce_mean/legacy_negative_axis,arch-a,PASS,PASS,0.050 s,0.00 MiB,0.00 MiB,6,0,SKIP,SKIP,SKIP,0.064 s,0.00 MiB,0.00 MiB,6,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
reduce_mean/legacy_reduce_all_keepdims_1,arch-a,PASS,PASS,0.040 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.041 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
reduce_mean/negative_axis,arch-a,PASS,PASS,0.044 s,0.00 MiB,0.00 MiB,6,0,SKIP,SKIP,SKIP,0.046 s,0.00 MiB,0.00 MiB,6,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
relu/4d,arch-a,PASS,PASS,0.043 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.039 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
relu/after_conv,arch-a,PASS,PASS,0.061 s,0.00 MiB,0.00 MiB,3,3,SKIP,SKIP,SKIP,0.062 s,0.00 MiB,0.00 MiB,4,3,SKIP,SKIP,SKIP,SKIP
|
||||||
|
relu/after_gemm,arch-a,PASS,PASS,0.044 s,0.01 MiB,0.01 MiB,4,4,SKIP,SKIP,SKIP,0.045 s,0.01 MiB,0.01 MiB,5,4,SKIP,SKIP,SKIP,SKIP
|
||||||
|
relu/basic,arch-a,PASS,PASS,0.039 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.038 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
reshape/4d_to_2d_flatten,arch-a,PASS,PASS,0.042 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.045 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
reshape/infer_dim_minus_one,arch-a,PASS,PASS,0.041 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.040 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
reshape/same_rank,arch-a,PASS,PASS,0.040 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.039 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
reshape/zero_copies_input_dim,arch-a,PASS,PASS,0.042 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.036 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
resize/height_only,arch-a,PASS,PASS,0.047 s,0.00 MiB,0.00 MiB,4,0,SKIP,SKIP,SKIP,0.047 s,0.00 MiB,0.00 MiB,4,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
resize/nearest_2x,arch-a,PASS,PASS,0.042 s,0.00 MiB,0.00 MiB,4,0,SKIP,SKIP,SKIP,0.051 s,0.00 MiB,0.00 MiB,4,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
resize/nearest_downsample,arch-a,PASS,PASS,0.042 s,0.00 MiB,0.00 MiB,2,0,SKIP,SKIP,SKIP,0.045 s,0.00 MiB,0.00 MiB,2,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
resize/non_uniform_scales,arch-a,PASS,PASS,0.042 s,0.00 MiB,0.00 MiB,6,0,SKIP,SKIP,SKIP,0.046 s,0.00 MiB,0.00 MiB,6,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
resize/width_only,arch-a,PASS,PASS,0.043 s,0.00 MiB,0.00 MiB,2,0,SKIP,SKIP,SKIP,0.043 s,0.00 MiB,0.00 MiB,2,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
resize/with_sizes,arch-a,PASS,PASS,0.045 s,0.00 MiB,0.00 MiB,3,0,SKIP,SKIP,SKIP,0.047 s,0.00 MiB,0.00 MiB,3,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
sigmoid/4d,arch-a,PASS,PASS,0.040 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.058 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
sigmoid/after_gemm,arch-a,PASS,PASS,0.048 s,0.01 MiB,0.01 MiB,4,4,SKIP,SKIP,SKIP,0.051 s,0.01 MiB,0.01 MiB,5,4,SKIP,SKIP,SKIP,SKIP
|
||||||
|
sigmoid/basic,arch-a,PASS,PASS,0.038 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.053 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
slice/2d_basic,arch-a,PASS,PASS,0.039 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.043 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
slice/after_conv,arch-a,PASS,PASS,0.050 s,0.00 MiB,0.01 MiB,6,6,SKIP,SKIP,SKIP,0.059 s,0.00 MiB,0.01 MiB,7,6,SKIP,SKIP,SKIP,SKIP
|
||||||
|
slice/default_axes,arch-a,PASS,PASS,0.039 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.038 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
slice/large_channel_1024,arch-a,PASS,PASS,0.041 s,0.01 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.039 s,0.01 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
slice/nchw_spatial_crop,arch-a,PASS,PASS,0.044 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.044 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
slice/negative_axis,arch-a,PASS,PASS,0.055 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.042 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
slice/negative_indices,arch-a,PASS,PASS,0.043 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.038 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
slice/step2,arch-a,PASS,PASS,0.041 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.042 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
softmax/3d_last_axis,arch-a,PASS,PASS,0.044 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.042 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
softmax/basic,arch-a,PASS,PASS,0.040 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.038 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
softmax/channel_axis,arch-a,PASS,PASS,0.037 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.047 s,0.00 MiB,0.00 MiB,3,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
softmax/large_dimension_1024,arch-a,PASS,PASS,0.036 s,0.01 MiB,0.01 MiB,1,0,SKIP,SKIP,SKIP,0.038 s,0.01 MiB,0.01 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
softmax/negative_axis,arch-a,PASS,PASS,0.069 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.060 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
split/basic,arch-a,PASS,PASS,0.052 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.057 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
split/equal_three_way,arch-a,PASS,PASS,0.042 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.042 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
split/negative_axis,arch-a,PASS,PASS,0.058 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.043 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
split/uneven_channel_axis_4d,arch-a,PASS,PASS,0.048 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.040 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
sub/after_gemm,arch-a,PASS,PASS,0.050 s,0.01 MiB,0.01 MiB,4,4,SKIP,SKIP,SKIP,0.047 s,0.01 MiB,0.01 MiB,5,4,SKIP,SKIP,SKIP,SKIP
|
||||||
|
sub/basic,arch-a,PASS,PASS,0.040 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.043 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
sub/broadcast_row,arch-a,PASS,PASS,0.056 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.044 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
sub/channel_broadcast_1024,arch-a,PASS,PASS,0.052 s,0.02 MiB,0.01 MiB,1,0,SKIP,SKIP,SKIP,0.043 s,0.02 MiB,0.01 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
sub/constant_lhs_broadcast,arch-a,PASS,PASS,0.044 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.038 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
sub/leading_dimension_broadcast,arch-a,PASS,PASS,0.040 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.033 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||||
|
@@ -3,7 +3,6 @@ from __future__ import annotations
|
|||||||
import argparse
|
import argparse
|
||||||
import copy
|
import copy
|
||||||
import csv
|
import csv
|
||||||
import importlib.util
|
|
||||||
import json
|
import json
|
||||||
import os
|
import os
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -48,26 +47,6 @@ def test_ablation_results_stay_in_independent_output_roots(tmp_path):
|
|||||||
assert rows[1]["latency"] == change
|
assert rows[1]["latency"] == change
|
||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("custom_input", [False, True])
|
|
||||||
def test_ablation_plots_follow_the_selected_experiment(tmp_path, monkeypatch, custom_input):
|
|
||||||
spec = importlib.util.spec_from_file_location("ablation_plots", TOOLS / "ablation/generate_plots.py")
|
|
||||||
plots = importlib.util.module_from_spec(spec)
|
|
||||||
monkeypatch.setitem(sys.modules, spec.name, plots)
|
|
||||||
spec.loader.exec_module(plots)
|
|
||||||
expected = tmp_path / "experiment/results_ablation.csv" if custom_input else (
|
|
||||||
ablation.DEFAULT_RESULTS_PATH
|
|
||||||
)
|
|
||||||
monkeypatch.setattr(sys, "argv", ["generate_plots.py", *(["--input", str(expected)] if custom_input else [])])
|
|
||||||
result = plots.Result("baseline", "arch-a", "vgg8", "latency", 1, "PASS", {})
|
|
||||||
inputs, outputs = [], []
|
|
||||||
monkeypatch.setattr(plots, "configure_style", lambda: None)
|
|
||||||
monkeypatch.setattr(plots, "load_results", lambda path: inputs.append(path) or [result])
|
|
||||||
monkeypatch.setattr(plots, "draw_metric", lambda rows, metric, label, output: outputs.append(output))
|
|
||||||
plots.main()
|
|
||||||
assert inputs == [expected]
|
|
||||||
assert outputs == [expected.parent / "plots"] * 3
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def recorded_failure(tmp_path):
|
def recorded_failure(tmp_path):
|
||||||
model, functional, config = (tmp_path / name for name in ("model.onnx", "functional.onnx", "config.json"))
|
model, functional, config = (tmp_path / name for name in ("model.onnx", "functional.onnx", "config.json"))
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
import csv
|
import csv
|
||||||
|
from dataclasses import replace
|
||||||
import importlib.util
|
import importlib.util
|
||||||
import math
|
import math
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
@@ -14,16 +15,14 @@ TOOLS = Path(__file__).resolve().parents[1] / "tools/pim"
|
|||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def plotters(monkeypatch):
|
def comparison(monkeypatch):
|
||||||
modules = []
|
directory = TOOLS / "pimcomp/compare"
|
||||||
for name, directory in (("comparison_plots", "pimcomp/compare"), ("ablation_plots", "ablation")):
|
monkeypatch.syspath_prepend(str(directory))
|
||||||
monkeypatch.syspath_prepend(str(TOOLS / directory))
|
spec = importlib.util.spec_from_file_location("comparison_plots", directory / "generate_plots.py")
|
||||||
spec = importlib.util.spec_from_file_location(name, TOOLS / directory / "generate_plots.py")
|
module = importlib.util.module_from_spec(spec)
|
||||||
module = importlib.util.module_from_spec(spec)
|
monkeypatch.setitem(sys.modules, spec.name, module)
|
||||||
monkeypatch.setitem(sys.modules, name, module)
|
spec.loader.exec_module(module)
|
||||||
spec.loader.exec_module(module)
|
return module
|
||||||
modules.append(module)
|
|
||||||
return modules
|
|
||||||
|
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
@@ -42,8 +41,7 @@ def write_csv(path, fields, rows):
|
|||||||
|
|
||||||
|
|
||||||
@pytest.mark.parametrize("custom", [False, True])
|
@pytest.mark.parametrize("custom", [False, True])
|
||||||
def test_comparison_plot_cli_input_paths(plotters, tmp_path, monkeypatch, custom):
|
def test_comparison_plot_cli_input_paths(comparison, tmp_path, monkeypatch, custom):
|
||||||
comparison, _ = plotters
|
|
||||||
expected = tmp_path / "custom.csv" if custom else TOOLS / "pimcomp/compare/results_comparison.csv"
|
expected = tmp_path / "custom.csv" if custom else TOOLS / "pimcomp/compare/results_comparison.csv"
|
||||||
seen = []
|
seen = []
|
||||||
|
|
||||||
@@ -62,8 +60,7 @@ def test_comparison_plot_cli_input_paths(plotters, tmp_path, monkeypatch, custom
|
|||||||
assert seen == [expected]
|
assert seen == [expected]
|
||||||
|
|
||||||
|
|
||||||
def test_exact_comparison_configuration_matrix(plotters, comparison_csv, tmp_path):
|
def test_exact_comparison_configuration_matrix(comparison, comparison_csv, tmp_path):
|
||||||
comparison, _ = plotters
|
|
||||||
fields, rows = comparison_csv
|
fields, rows = comparison_csv
|
||||||
results = comparison.load_results(write_csv(tmp_path / "results.csv", fields, rows))
|
results = comparison.load_results(write_csv(tmp_path / "results.csv", fields, rows))
|
||||||
keys = {(row.arch, row.model, row.mode, row.pipeline) for row in results}
|
keys = {(row.arch, row.model, row.mode, row.pipeline) for row in results}
|
||||||
@@ -74,8 +71,7 @@ def test_exact_comparison_configuration_matrix(plotters, comparison_csv, tmp_pat
|
|||||||
@pytest.mark.parametrize("damage,message", [("missing", "Missing"), ("duplicate", "Duplicate"),
|
@pytest.mark.parametrize("damage,message", [("missing", "Missing"), ("duplicate", "Duplicate"),
|
||||||
("arch", "Unexpected"), ("model", "Unexpected"),
|
("arch", "Unexpected"), ("model", "Unexpected"),
|
||||||
("pipeline", "Unexpected"), ("mode", "Unexpected")])
|
("pipeline", "Unexpected"), ("mode", "Unexpected")])
|
||||||
def test_comparison_rejects_bad_matrix_keys(plotters, comparison_csv, tmp_path, damage, message):
|
def test_comparison_rejects_bad_matrix_keys(comparison, comparison_csv, tmp_path, damage, message):
|
||||||
comparison, _ = plotters
|
|
||||||
fields, rows = comparison_csv
|
fields, rows = comparison_csv
|
||||||
if damage == "missing":
|
if damage == "missing":
|
||||||
rows.pop()
|
rows.pop()
|
||||||
@@ -108,50 +104,431 @@ else:
|
|||||||
@pytest.mark.parametrize("field,value", [("pimcomp_latency_ms", "999"),
|
@pytest.mark.parametrize("field,value", [("pimcomp_latency_ms", "999"),
|
||||||
("pimcomp_throughput_samples_s", "999"), ("pimcomp_power_mw", "999"),
|
("pimcomp_throughput_samples_s", "999"), ("pimcomp_power_mw", "999"),
|
||||||
("pimcomp_energy_pj", "999"), ("pimcomp_functional_validation", "FAIL")])
|
("pimcomp_energy_pj", "999"), ("pimcomp_functional_validation", "FAIL")])
|
||||||
def test_repeated_pimcomp_batch_results_must_agree(plotters, comparison_csv, tmp_path, field, value):
|
def test_repeated_pimcomp_batch_results_must_agree(comparison, comparison_csv, tmp_path, field, value):
|
||||||
comparison, _ = plotters
|
|
||||||
fields, rows = comparison_csv
|
fields, rows = comparison_csv
|
||||||
row = next(row for row in rows if row["arch"] == "arch-a" and row["model"] == "vgg8"
|
row = next(row for row in rows if row["arch"] == "arch-a" and row["model"] == "vgg8"
|
||||||
and row["raptor_pipeline"] == "4")
|
and row["raptor_pipeline"] == "4")
|
||||||
row[field] = value
|
row[field] = value
|
||||||
with pytest.raises(ValueError, match="inconsistent shared Pimcomp batch results"):
|
with pytest.raises(ValueError, match="inconsistent shared PIMCOMP batch results"):
|
||||||
comparison.load_results(write_csv(tmp_path / "results.csv", fields, rows))
|
comparison.load_results(write_csv(tmp_path / "results.csv", fields, rows))
|
||||||
|
|
||||||
|
|
||||||
def test_legacy_invalid_ablation_pairs_are_unavailable_without_changing_pass(plotters):
|
@pytest.mark.parametrize("status,expected", [("PASS", ["1.0"]),
|
||||||
_, ablation = plotters
|
("FAIL", ["1.0", "INVALID"]), ("SKIPPED", ["1.0", "SKIPPED"]),
|
||||||
with pytest.warns(UserWarning, match="2 invalid reciprocal"):
|
("NA", ["1.0", "NA"])])
|
||||||
rows = ablation.load_results(TOOLS / "ablation/results_ablation.csv")
|
@pytest.mark.parametrize("direction", [1, -1])
|
||||||
invalid = [row for row in rows if row.model == "googlenet" and row.variant == "no-plan"
|
def test_value_label_validation_text(comparison, status, expected, direction):
|
||||||
and row.mode == "throughput"]
|
fig, ax = comparison.plt.subplots()
|
||||||
assert len(invalid) == 2
|
try:
|
||||||
assert all(row.validation == "PASS" and math.isnan(row.metrics["latency"])
|
comparison.value_label(ax, 0, 1, "1.0", status, direction)
|
||||||
and math.isnan(row.metrics["throughput"]) for row in invalid)
|
assert [text.get_text() for text in ax.texts] == expected
|
||||||
|
assert ax.texts[-1].get_color() == (comparison.ERROR_COLOR if status == "FAIL" else comparison.TEXT_COLOR)
|
||||||
|
if status != "PASS":
|
||||||
|
value, validation = ax.texts
|
||||||
|
assert value.xy == validation.xy
|
||||||
|
assert value.get_position()[1] == validation.get_position()[1] == 0
|
||||||
|
assert value.get_position()[0] == -validation.get_position()[0] != 0
|
||||||
|
finally:
|
||||||
|
comparison.plt.close(fig)
|
||||||
|
|
||||||
|
|
||||||
def test_ablation_missing_values_and_legitimate_extremes(plotters, tmp_path):
|
@pytest.mark.parametrize("values", [(1, 2), (-1, -2), (-1, 2), (0, 0)])
|
||||||
_, ablation = plotters
|
def test_value_label_limits_are_tight_shared_and_unclipped(comparison, values):
|
||||||
row = {"variant": "no-plan", "arch": "arch-a", "model": "vgg8", "mode": "throughput",
|
fig, axes = comparison.plt.subplots(1, 2, sharey=True, figsize=(10, 4.4), layout="constrained")
|
||||||
"raptor_pipeline": "4", "functional_validation": "FAIL", "latency": "-100.00%",
|
try:
|
||||||
"throughput": "+100000000.00%", "power": "NA", "energy": "-100.00%"}
|
for ax, value in zip(axes, values):
|
||||||
result = ablation.load_results(write_csv(tmp_path / "extreme.csv", list(row), [row]))[0]
|
ax.set_ylim(-10, 10)
|
||||||
assert result.validation == "FAIL"
|
ax.bar(0, value)
|
||||||
assert result.metrics["latency"] == result.metrics["energy"] == -100
|
direction = -1 if value < 0 else 1
|
||||||
assert result.metrics["throughput"] == 100000000
|
comparison.value_label(ax, 0, value + direction * 0.05, f"{value:+.2f}%", "FAIL", direction)
|
||||||
assert math.isnan(result.metrics["power"])
|
comparison.fit_value_labels(fig)
|
||||||
|
fig.canvas.draw()
|
||||||
|
renderer = fig.canvas.get_renderer()
|
||||||
|
assert axes[0].get_ylim() == axes[1].get_ylim()
|
||||||
|
lower, upper = axes[0].get_ylim()
|
||||||
|
assert upper - lower < 20
|
||||||
|
assert lower <= min(0, *values) and upper >= max(0, *values)
|
||||||
|
assert axes[0].get_window_extent(renderer).height == pytest.approx(axes[1].get_window_extent(renderer).height)
|
||||||
|
for ax in axes:
|
||||||
|
bounds = ax.get_window_extent(renderer)
|
||||||
|
for text in ax.texts:
|
||||||
|
extent = text.get_window_extent(renderer)
|
||||||
|
assert bounds.y0 <= extent.y0 <= extent.y1 <= bounds.y1
|
||||||
|
finally:
|
||||||
|
comparison.plt.close(fig)
|
||||||
|
|
||||||
|
|
||||||
def test_both_plotters_export_png_and_svg(plotters, tmp_path):
|
@pytest.mark.parametrize("statuses,expected", [
|
||||||
comparison, ablation = plotters
|
(("PASS", "PASS", "PASS"), "PASS"),
|
||||||
|
(("FAIL", "PASS", "PASS"), "FAIL"),
|
||||||
|
(("PASS", "FAIL", "PASS"), "FAIL"),
|
||||||
|
(("PASS", "PASS", "FAIL"), "FAIL"),
|
||||||
|
(("PASS", "SKIPPED", "NA"), "SKIPPED"),
|
||||||
|
(("PASS", "PASS", "NA"), "N/A"),
|
||||||
|
])
|
||||||
|
def test_compact_validation_combines_pipeline_statuses(comparison, tmp_path, monkeypatch, statuses, expected):
|
||||||
|
rows = comparison.load_results(TOOLS / "pimcomp/compare/results_comparison.csv")
|
||||||
|
stages = dict(zip((2, 4, 8), statuses))
|
||||||
|
rows = [replace(row, raptor_validation=stages[row.pipeline])
|
||||||
|
if row.arch == "arch-a" and row.model == "vgg8" and row.mode == "throughput"
|
||||||
|
else row for row in rows]
|
||||||
|
captured = []
|
||||||
|
monkeypatch.setattr(comparison, "save_figure", lambda fig, name, path: captured.append((fig, name)))
|
||||||
|
comparison.draw_validation(rows, tmp_path, compact=True)
|
||||||
|
fig, name = captured[0]
|
||||||
|
try:
|
||||||
|
assert name == "validation_status_compact"
|
||||||
|
assert fig.get_size_inches() == pytest.approx((8.5, 3.3))
|
||||||
|
for index, ax in enumerate(fig.axes):
|
||||||
|
assert [label.get_text() for label in ax.get_xticklabels(minor=True)] == [
|
||||||
|
"Raptor", "PIMCOMP", "Raptor", "PIMCOMP"]
|
||||||
|
assert [label.get_text() for label in ax.get_yticklabels()] == ([
|
||||||
|
"VGG-8", "ResNet-18", "ResNet-34", "GoogLeNet", "YOLO11n"] if index == 0 else [])
|
||||||
|
assert len(ax.patches) == len(ax.texts) == 4 * len(comparison.MODELS)
|
||||||
|
assert [label.get_text() for label in ax.get_xticklabels()] == ["Latency", "Throughput"]
|
||||||
|
assert all(text.get_fontsize() == 12 and text.get_weight() == "normal" for text in ax.texts)
|
||||||
|
assert all(text.get_color() == "#1D2530" for text in ax.texts)
|
||||||
|
ax = fig.axes[0]
|
||||||
|
throughput_x = ax.xaxis.get_minorticklocs()[2]
|
||||||
|
assert [text.get_text() for text in ax.texts if text.get_position() == (throughput_x, 0)] == [expected]
|
||||||
|
finally:
|
||||||
|
comparison.plt.close(fig)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("rates,stage", [((300, 200, 100), 2), ((100, 300, 200), 4),
|
||||||
|
((100, 200, 300), 8), ((math.nan, 200, 300), 8), ((300, 300, 200), 2),
|
||||||
|
((math.nan, math.nan, math.nan), 2)])
|
||||||
|
def test_compact_speedups_select_best_pipeline(comparison, rates, stage):
|
||||||
|
template = comparison.load_results(TOOLS / "pimcomp/compare/results_comparison.csv")[0]
|
||||||
|
rows = [replace(template, latency_speedup=0.5)]
|
||||||
|
rows += [replace(template, mode="throughput", pipeline=depth, raptor_throughput=rate,
|
||||||
|
pimcomp_throughput=100, raptor_validation="FAIL")
|
||||||
|
for depth, rate in zip((2, 4, 8), rates)]
|
||||||
|
latency, throughput = comparison.select_compact_speedups(rows)
|
||||||
|
assert latency is rows[0] and latency.speedup == 0.5
|
||||||
|
assert throughput.pipeline == stage and throughput.raptor_validation == "FAIL"
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("diverging", [False, True])
|
||||||
|
@pytest.mark.parametrize("metric,ylabel", [("speedup", "Speedup (x)"),
|
||||||
|
("energy", "Energy-per-inference advantage (x)"), ("power", "Average power advantage (x)")])
|
||||||
|
def test_compact_metric_bars_and_units(comparison, tmp_path, monkeypatch, diverging, metric, ylabel):
|
||||||
|
from matplotlib.colors import to_rgba
|
||||||
|
|
||||||
|
template = comparison.load_results(TOOLS / "pimcomp/compare/results_comparison.csv")[0]
|
||||||
|
rows = [replace(template, arch=arch, mode=mode, pipeline=stage, latency_speedup=0.5,
|
||||||
|
energy=0.5 if mode == "latency" else 3, power=0.5 if mode == "latency" else 3,
|
||||||
|
raptor_throughput=300, pimcomp_throughput=100, pimcomp_validation="FAIL")
|
||||||
|
for arch in comparison.ARCHITECTURES for mode, stage in (("latency", 1), ("throughput", 4))]
|
||||||
|
captured = []
|
||||||
|
monkeypatch.setattr(comparison, "save_figure", lambda fig, name, path: captured.append(fig))
|
||||||
|
comparison.draw_metric(rows, metric, ylabel, "compact", tmp_path,
|
||||||
|
diverging=diverging, compact=True)
|
||||||
|
fig = captured[0]
|
||||||
|
try:
|
||||||
|
assert fig.get_size_inches() == pytest.approx((8.5, 4.4))
|
||||||
|
expected_legend = {
|
||||||
|
"speedup": ["Single-inference latency", "Best-case throughput"],
|
||||||
|
"energy": ["Single-inference energy", "Best-case throughput energy"],
|
||||||
|
"power": ["Single-inference power", "Best-case throughput power"],
|
||||||
|
}
|
||||||
|
assert [text.get_text() for text in fig.legends[0].get_texts()] == expected_legend[metric]
|
||||||
|
for index, ax in enumerate(fig.axes):
|
||||||
|
latency, throughput = (container.patches[0] for container in ax.containers)
|
||||||
|
lower, upper = ax.get_xlim()
|
||||||
|
assert latency.get_x() - lower == pytest.approx(0.4)
|
||||||
|
assert upper - throughput.get_x() - throughput.get_width() == pytest.approx(0.4)
|
||||||
|
assert latency.get_x() + latency.get_width() == throughput.get_x()
|
||||||
|
assert latency.get_facecolor() == to_rgba(comparison.METRIC_STYLES["latency"][0])
|
||||||
|
assert throughput.get_facecolor() == to_rgba(comparison.METRIC_STYLES["throughput"][0])
|
||||||
|
assert latency.get_edgecolor() == to_rgba(comparison.METRIC_STYLES["latency"][1])
|
||||||
|
assert throughput.get_edgecolor() == to_rgba(comparison.METRIC_STYLES["throughput"][1])
|
||||||
|
assert not latency.get_hatch() and throughput.get_hatch() == "//"
|
||||||
|
assert latency.get_height() == (-1 if diverging else 0.5)
|
||||||
|
assert throughput.get_height() == (2 if diverging else 3)
|
||||||
|
assert ax.get_ylabel() == (ylabel if index == 0 else "")
|
||||||
|
assert bool(ax.get_yticklabels()) == (index == 0)
|
||||||
|
assert all("×" not in label.get_text() for label in ax.get_yticklabels())
|
||||||
|
assert [text.get_text() for text in ax.texts] == [
|
||||||
|
"2.00×" if diverging else "0.50×", "P. INVALID", "3.00×", "P. INVALID"]
|
||||||
|
finally:
|
||||||
|
comparison.plt.close(fig)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("missing_energy", [False, True])
|
||||||
|
def test_compact_metrics_use_speedup_pipeline_selection(comparison, monkeypatch, missing_energy):
|
||||||
|
rows = comparison.load_results(TOOLS / "pimcomp/compare/results_comparison.csv")
|
||||||
|
cases = {2: (100, 9, 8), 4: (300, math.nan if missing_energy else 0.5, 0.6), 8: (200, 7, 6)}
|
||||||
|
rows = [replace(row, raptor_throughput=cases[row.pipeline][0],
|
||||||
|
energy=cases[row.pipeline][1], power=cases[row.pipeline][2])
|
||||||
|
if row.arch == "arch-a" and row.model == "vgg8" and row.mode == "throughput"
|
||||||
|
else row for row in rows]
|
||||||
|
captured = []
|
||||||
|
|
||||||
|
def capture(selected, field, ylabel, basename, output_dir, diverging=False, compact=False):
|
||||||
|
if compact:
|
||||||
|
captured.append((selected, field, basename, diverging))
|
||||||
|
|
||||||
|
monkeypatch.setattr(comparison, "load_results", lambda path: rows)
|
||||||
|
for name in ("configure_style", "draw_validation", "draw_pipeline_effect", "draw_absolute"):
|
||||||
|
monkeypatch.setattr(comparison, name, lambda *args, **kwargs: None)
|
||||||
|
monkeypatch.setattr(comparison, "draw_metric", capture)
|
||||||
|
monkeypatch.setattr(sys, "argv", ["generate_plots.py"])
|
||||||
|
comparison.main()
|
||||||
|
assert {(field, basename, diverging) for _, field, basename, diverging in captured} == {
|
||||||
|
(metric, f"{metric}_compact_{suffix}", diverging)
|
||||||
|
for metric in ("speedup", "energy", "power")
|
||||||
|
for suffix, diverging in (("comparison", False), ("diverging", True))}
|
||||||
|
assert len(captured) == 6
|
||||||
|
selected = captured[0][0]
|
||||||
|
assert all(plotted is selected for plotted, _, _, _ in captured)
|
||||||
|
throughput = next(row for row in selected if row.arch == "arch-a" and row.model == "vgg8"
|
||||||
|
and row.mode == "throughput")
|
||||||
|
assert throughput.pipeline == 4 and throughput.power == 0.6
|
||||||
|
assert math.isnan(throughput.energy) if missing_energy else throughput.energy == 0.5
|
||||||
|
|
||||||
|
|
||||||
|
def test_compiler_palette_and_grayscale_patterns(comparison, tmp_path, monkeypatch):
|
||||||
|
from matplotlib.colors import to_rgba
|
||||||
|
|
||||||
|
assert comparison.COMPILER_STYLES == {
|
||||||
|
"raptor": ("#B3D5CA", "#4B7C6A"), "pimcomp": ("#E1BC8B", "#856A44")}
|
||||||
|
assert comparison.METRIC_STYLES == {
|
||||||
|
"latency": ("#B3CCE3", "#456B90"), "throughput": ("#B3D5CA", "#4B7C6A")}
|
||||||
|
assert comparison.VALIDATION_STYLES["PASS"] == ("#B3D5CA", "#4B7C6A")
|
||||||
|
assert comparison.TEXT_COLOR == "#222222"
|
||||||
|
assert comparison.NETWORK_STYLES["yolo11n"] == ("#FF9494", "#C52020")
|
||||||
|
assert comparison.NETWORK_STYLES["googlenet"] == ("#D1C0E0", "#77558F")
|
||||||
|
rows = comparison.load_results(TOOLS / "pimcomp/compare/results_comparison.csv")
|
||||||
|
selected = [replace(row, power=2 if row.arch == "arch-a" else 0.5)
|
||||||
|
for row in rows if row.model == "vgg8" and row.mode == "latency"]
|
||||||
|
captured = []
|
||||||
|
monkeypatch.setattr(comparison, "save_figure", lambda fig, *args: captured.append(fig))
|
||||||
|
comparison.draw_metric(selected, "power", "Average power advantage (x)", "power", tmp_path, diverging=True)
|
||||||
|
comparison.draw_absolute(selected, "latency", "Single-inference latency (ms)", "latency", tmp_path)
|
||||||
|
try:
|
||||||
|
for ax, compiler in zip(captured[0].axes, ("raptor", "pimcomp")):
|
||||||
|
bar = ax.containers[0].patches[0]
|
||||||
|
assert bar.get_facecolor() == to_rgba(comparison.COMPILER_STYLES[compiler][0])
|
||||||
|
assert bar.get_edgecolor() == to_rgba(comparison.COMPILER_STYLES[compiler][1])
|
||||||
|
assert not bar.get_hatch()
|
||||||
|
for ax in captured[1].axes:
|
||||||
|
assert [container.patches[0].get_hatch() for container in ax.containers] == ["", ""]
|
||||||
|
finally:
|
||||||
|
for fig in captured:
|
||||||
|
comparison.plt.close(fig)
|
||||||
|
|
||||||
|
|
||||||
|
def test_tick_rotation_changes_only_overlapping_labels(comparison):
|
||||||
|
fig, ax = comparison.plt.subplots(figsize=(4, 3), layout="constrained")
|
||||||
|
try:
|
||||||
|
ax.set_xlim(-1, 6)
|
||||||
|
ax.set_xticks([0, 0.5, 5], ["Single-inference", "Throughput", "8"])
|
||||||
|
comparison.fit_x_tick_labels(fig)
|
||||||
|
assert [text.get_rotation() for text in ax.get_xticklabels()] == [330, 330, 0]
|
||||||
|
assert [text.get_ha() for text in ax.get_xticklabels()] == ["left", "left", "center"]
|
||||||
|
finally:
|
||||||
|
comparison.plt.close(fig)
|
||||||
|
|
||||||
|
|
||||||
|
def test_all_plot_terminology(comparison, monkeypatch):
|
||||||
|
from matplotlib.path import Path as PlotPath
|
||||||
|
from matplotlib.text import Text
|
||||||
|
from matplotlib.transforms import Affine2D
|
||||||
|
|
||||||
|
plotter = comparison
|
||||||
|
labels = set()
|
||||||
|
|
||||||
|
def capture(fig, name, output):
|
||||||
|
fig.canvas.draw()
|
||||||
|
if name != "raptor_pipeline_throughput":
|
||||||
|
assert fig.axes[1].get_ylabel() == ""
|
||||||
|
assert fig.axes[1].get_yticklabels() == []
|
||||||
|
renderer = fig.canvas.get_renderer()
|
||||||
|
for ax in fig.axes:
|
||||||
|
for container in ax.containers:
|
||||||
|
for bar in container.patches:
|
||||||
|
center = bar.get_x() + bar.get_width() / 2
|
||||||
|
bounds = bar.get_window_extent(renderer)
|
||||||
|
for text in ax.texts:
|
||||||
|
if hasattr(text, "xy") and text.xy[0] == center:
|
||||||
|
extent = text.get_window_extent(renderer)
|
||||||
|
assert bounds.x0 <= extent.x0 <= extent.x1 <= bounds.x1, (name, text.get_text())
|
||||||
|
for legend in fig.legends:
|
||||||
|
bounds = legend.get_window_extent(renderer)
|
||||||
|
assert fig.bbox.x0 <= bounds.x0 <= bounds.x1 <= fig.bbox.x1, name
|
||||||
|
tick_paths = []
|
||||||
|
for ax in fig.axes:
|
||||||
|
for minor in (False, True):
|
||||||
|
for text in ax.get_xticklabels(minor=minor):
|
||||||
|
if not text.get_visible() or not text.get_text():
|
||||||
|
continue
|
||||||
|
angle = text.get_rotation()
|
||||||
|
assert angle in (0, 330), (name, text.get_text(), angle)
|
||||||
|
bounds = text.get_window_extent(renderer)
|
||||||
|
text.set_rotation(0)
|
||||||
|
unrotated = text.get_window_extent(renderer)
|
||||||
|
text.set_rotation(angle)
|
||||||
|
path = PlotPath.unit_rectangle().transformed(
|
||||||
|
Affine2D().scale(unrotated.width, unrotated.height).rotate_deg(angle))
|
||||||
|
rotated = path.get_extents()
|
||||||
|
path = path.transformed(Affine2D().translate(bounds.x0 - rotated.x0, bounds.y0 - rotated.y0))
|
||||||
|
assert not any(path.intersects_path(other, filled=True) for other in tick_paths), name
|
||||||
|
tick_paths.append(path)
|
||||||
|
if text.get_text() in comparison.MODEL_NAMES.values():
|
||||||
|
assert angle == 0, (name, text.get_text())
|
||||||
|
heights = [ax.get_window_extent(renderer).height for ax in fig.axes]
|
||||||
|
assert max(heights) - min(heights) < 1, name
|
||||||
|
for ax in fig.axes:
|
||||||
|
bounds = ax.get_window_extent(renderer)
|
||||||
|
for text in ax.texts:
|
||||||
|
extent = text.get_window_extent(renderer)
|
||||||
|
assert bounds.y0 - 1 <= extent.y0 <= extent.y1 <= bounds.y1 + 1, (name, text.get_text())
|
||||||
|
assert fig.bbox.x0 <= extent.x0 <= extent.x1 <= fig.bbox.x1, (name, text.get_text())
|
||||||
|
labels.update(text.get_text() for text in fig.findobj(Text))
|
||||||
|
plotter.plt.close(fig)
|
||||||
|
|
||||||
|
monkeypatch.setattr(plotter, "save_figure", capture)
|
||||||
|
monkeypatch.setattr(sys, "argv", ["generate_plots.py"])
|
||||||
|
plotter.main()
|
||||||
|
text = "\n".join(labels)
|
||||||
|
for old in ("Pimcomp", "ARCHITECTURE", "VGG8", "RESNET", "GOOGLENET", "YOLO11N",
|
||||||
|
"Single inference", "Best case", "Latency, 1 stage", "Energy per inference advantage"):
|
||||||
|
assert old not in text
|
||||||
|
assert {"VGG-8", "ResNet-18", "ResNet-34", "GoogLeNet"} <= labels
|
||||||
|
assert "Arch-A" in text and "Arch-B" in text
|
||||||
|
assert {"Raptor", "PIMCOMP", "YOLO11n", "Single-inference", "2 stages", "4 stages", "8 stages",
|
||||||
|
"Single-inference latency (ms)", "Throughput (inferences/s)", "Energy per inference (mJ)",
|
||||||
|
"Energy-per-inference advantage (x)", "Derived as 1 / latency",
|
||||||
|
"Throughput measurement"} <= labels
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("width,one_row", [(12, True), (4.5, False), (3, False)])
|
||||||
|
def test_legend_uses_available_width_and_balances_rows(comparison, width, one_row):
|
||||||
|
from collections import Counter
|
||||||
|
from matplotlib.patches import Patch
|
||||||
|
|
||||||
|
fig = comparison.plt.figure(figsize=(width, 4), layout="constrained")
|
||||||
|
try:
|
||||||
|
handles = [Patch(label=f"Entry {index}") for index in range(7)]
|
||||||
|
comparison.add_legend(fig, handles)
|
||||||
|
fig.canvas.draw()
|
||||||
|
renderer = fig.canvas.get_renderer()
|
||||||
|
legend = fig.legends[0]
|
||||||
|
counts = Counter(round(text.get_window_extent(renderer).y0, 3)
|
||||||
|
for text in legend.get_texts() if text.get_visible())
|
||||||
|
assert sum(counts.values()) == len(handles)
|
||||||
|
assert (len(counts) == 1) == one_row
|
||||||
|
assert max(counts.values()) - min(counts.values()) <= 1
|
||||||
|
bounds = legend.get_window_extent(renderer)
|
||||||
|
assert 0 <= bounds.x0 <= bounds.x1 <= fig.bbox.width
|
||||||
|
finally:
|
||||||
|
comparison.plt.close(fig)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.parametrize("diverging", [False, True])
|
||||||
|
def test_advantage_throughput_groups_touch(comparison, tmp_path, monkeypatch, diverging):
|
||||||
|
rows = comparison.load_results(TOOLS / "pimcomp/compare/results_comparison.csv")
|
||||||
|
selected = [row for row in rows if row.model in ("vgg8", "resnet18")]
|
||||||
|
captured = []
|
||||||
|
monkeypatch.setattr(comparison, "save_figure", lambda fig, *args: captured.append(fig))
|
||||||
|
comparison.draw_metric(selected, "energy", "Energy-per-inference advantage (x)", "energy", tmp_path,
|
||||||
|
diverging=diverging)
|
||||||
|
fig = captured[0]
|
||||||
|
try:
|
||||||
|
for ax in fig.axes:
|
||||||
|
bars = [container.patches[0] for container in ax.containers]
|
||||||
|
lower, upper = ax.get_xlim()
|
||||||
|
assert bars[0].get_x() - lower == pytest.approx(0.4)
|
||||||
|
assert upper - bars[-1].get_x() - bars[-1].get_width() == pytest.approx(0.4)
|
||||||
|
for index, (left, right) in enumerate(zip(bars, bars[1:])):
|
||||||
|
edge = left.get_x() + left.get_width()
|
||||||
|
if index % 4 in (1, 2):
|
||||||
|
assert edge == pytest.approx(right.get_x())
|
||||||
|
else:
|
||||||
|
assert edge < right.get_x()
|
||||||
|
finally:
|
||||||
|
comparison.plt.close(fig)
|
||||||
|
|
||||||
|
|
||||||
|
def test_grouped_absolute_plot_scheduling_labels(comparison, tmp_path, monkeypatch):
|
||||||
|
rows = comparison.load_results(TOOLS / "pimcomp/compare/results_comparison.csv")
|
||||||
|
captured = []
|
||||||
|
monkeypatch.setattr(comparison, "save_figure", lambda fig, *args: captured.append(fig))
|
||||||
|
comparison.draw_absolute(rows, "energy", "Energy per inference (mJ)", "energy", tmp_path, scale=1e-9)
|
||||||
|
fig = captured[0]
|
||||||
|
try:
|
||||||
|
for ax in fig.axes:
|
||||||
|
assert {text.get_text() for text in ax.get_xticklabels(minor=True)} == {"Latency", "Throughput"}
|
||||||
|
labels = [text.get_text() for text in fig.legends[0].get_texts()]
|
||||||
|
assert "1 stage (latency)" in labels and "Single-inference" not in labels
|
||||||
|
finally:
|
||||||
|
comparison.plt.close(fig)
|
||||||
|
|
||||||
|
|
||||||
|
def test_pipeline_plot_connects_derived_and_measured_rates(comparison, tmp_path, monkeypatch):
|
||||||
|
rows = comparison.load_results(TOOLS / "pimcomp/compare/results_comparison.csv")
|
||||||
|
captured = []
|
||||||
|
monkeypatch.setattr(comparison, "save_figure", lambda fig, *args: captured.append(fig))
|
||||||
|
comparison.draw_pipeline_effect(rows, tmp_path)
|
||||||
|
fig = captured[0]
|
||||||
|
try:
|
||||||
|
assert fig.get_size_inches() == pytest.approx((13, 4.3))
|
||||||
|
assert fig._supylabel.get_text() == "Throughput (inferences/s)"
|
||||||
|
for index, (ax, (arch, model)) in enumerate(zip(fig.axes, ((arch, model) for arch in comparison.ARCHITECTURES
|
||||||
|
for model in comparison.MODELS))):
|
||||||
|
selected = [row for row in rows if row.arch == arch and row.model == model]
|
||||||
|
measured_line, derived_marker, *measured_markers = ax.lines
|
||||||
|
assert measured_line.get_ydata()[0] == pytest.approx(1000 / selected[0].raptor_latency)
|
||||||
|
assert list(measured_line.get_ydata()[1:]) == [row.raptor_throughput for row in selected[1:]]
|
||||||
|
assert derived_marker.get_marker() == "s"
|
||||||
|
assert derived_marker.get_ydata()[0] == pytest.approx(1000 / selected[0].raptor_latency)
|
||||||
|
assert derived_marker.get_color() == comparison.NETWORK_STYLES[model][1]
|
||||||
|
assert all(marker.get_marker() == "o" for marker in measured_markers)
|
||||||
|
assert measured_line.get_color() == comparison.NETWORK_STYLES[model][1]
|
||||||
|
assert [label.get_text() for label in ax.get_xticklabels()] == (
|
||||||
|
["1", "2", "4", "8"] if arch == "arch-b" else [])
|
||||||
|
assert ax.get_xlabel() == ""
|
||||||
|
assert ax.get_title() == (comparison.MODEL_NAMES[model] if arch == "arch-a" else "")
|
||||||
|
assert ax.title.get_weight() == "normal"
|
||||||
|
if model == comparison.MODELS[-1]:
|
||||||
|
assert ax.get_ylabel() == comparison.ARCH_NAMES[arch]
|
||||||
|
assert ax.yaxis.get_label_position() == "right"
|
||||||
|
assert ax.yaxis.label.get_rotation() == 270
|
||||||
|
assert ax.yaxis.label.get_weight() == "bold"
|
||||||
|
else:
|
||||||
|
assert ax.get_ylabel() == ""
|
||||||
|
assert all(label.get_rotation() == 0 for label in ax.get_xticklabels())
|
||||||
|
lower, upper = ax.get_ylim()
|
||||||
|
values = measured_line.get_ydata()
|
||||||
|
assert 0 < lower < min(values) <= max(values) < upper
|
||||||
|
assert ax.get_yticklabels()
|
||||||
|
assert ax.get_ylim() == fig.axes[index % len(comparison.MODELS)].get_ylim()
|
||||||
|
assert [text.get_text() for text in fig.legends[0].get_texts()] == [
|
||||||
|
"Derived as 1 / latency", "Throughput measurement"]
|
||||||
|
assert len({ax.lines[0].get_color() for ax in fig.axes[:len(comparison.MODELS)]}) == len(comparison.MODELS)
|
||||||
|
finally:
|
||||||
|
comparison.plt.close(fig)
|
||||||
|
|
||||||
|
|
||||||
|
def test_plotter_exports_png_svg_and_pdf(comparison, tmp_path):
|
||||||
comparison.configure_style()
|
comparison.configure_style()
|
||||||
rows = comparison.load_results(TOOLS / "pimcomp/compare/results_comparison.csv")
|
rows = comparison.load_results(TOOLS / "pimcomp/compare/results_comparison.csv")
|
||||||
selected = [row for row in rows if row.model == "vgg8" and row.mode == "latency"]
|
selected = [replace(row, raptor_validation="FAIL", pimcomp_latency=math.nan)
|
||||||
|
for row in rows if row.model == "vgg8" and row.mode == "latency"]
|
||||||
comparison.draw_absolute(selected, "latency", "Latency (ms)", "comparison-smoke", tmp_path)
|
comparison.draw_absolute(selected, "latency", "Latency (ms)", "comparison-smoke", tmp_path)
|
||||||
row = ablation.Result("baseline", "arch-a", "vgg8", "latency", 1, "PASS",
|
comparison.draw_validation(rows, tmp_path, compact=True)
|
||||||
{"latency": math.nan, "power": 0, "energy": 0, "throughput": math.nan})
|
compact_names = []
|
||||||
ablation.draw_metric([row], "latency", "Latency change (%)", tmp_path)
|
for metric in ("speedup", "energy", "power"):
|
||||||
for name in ("comparison-smoke", "latency_pipeline1_latency"):
|
for suffix, diverging in (("comparison", False), ("diverging", True)):
|
||||||
|
name = f"{metric}_compact_{suffix}"
|
||||||
|
compact_names.append(name)
|
||||||
|
comparison.draw_metric(comparison.select_compact_speedups(rows), metric, "Advantage (x)",
|
||||||
|
name, tmp_path, diverging=diverging, compact=True)
|
||||||
|
for name in ("comparison-smoke", "validation_status_compact", *compact_names):
|
||||||
assert (tmp_path / f"png/{name}.png").read_bytes().startswith(b"\x89PNG\r\n\x1a\n")
|
assert (tmp_path / f"png/{name}.png").read_bytes().startswith(b"\x89PNG\r\n\x1a\n")
|
||||||
svg = (tmp_path / f"svg/{name}.svg").read_text()
|
svg = (tmp_path / f"svg/{name}.svg").read_text()
|
||||||
assert "<svg" in svg
|
assert "<svg" in svg
|
||||||
assert "N/A" in (tmp_path / "svg/latency_pipeline1_latency.svg").read_text()
|
assert (tmp_path / f"pdf/{name}.pdf").read_bytes().startswith(b"%PDF-")
|
||||||
|
assert "N/A" in (tmp_path / "svg/comparison-smoke.svg").read_text()
|
||||||
|
assert "P. INVALID" in (tmp_path / "svg/comparison-smoke.svg").read_text()
|
||||||
|
|||||||
@@ -47,27 +47,9 @@ guarantees, so those variants may fail numerical validation and exit nonzero.
|
|||||||
Baseline and `no-plan` must pass. Percentages from failing variants are not
|
Baseline and `no-plan` must pass. Percentages from failing variants are not
|
||||||
valid speedups; deadline averages do not prove steady-state convergence.
|
valid speedups; deadline averages do not prove steady-state convergence.
|
||||||
|
|
||||||
## Plots
|
## Result data
|
||||||
|
|
||||||
```bash
|
The checked-in `results_ablation.csv` is an intentional result dataset for tabular
|
||||||
.venv/bin/python validation/tools/pim/ablation/generate_plots.py
|
reporting. Ablation experiments and aggregation remain available through the
|
||||||
# For a custom experiment root:
|
runner above. Percentages from failing variants are not valid speedups. See the
|
||||||
.venv/bin/python validation/tools/pim/ablation/generate_plots.py \
|
|
||||||
--input /path/to/experiment/results_ablation.csv
|
|
||||||
```
|
|
||||||
|
|
||||||
Requires Matplotlib from the root `requirements.txt`. By default, reads the
|
|
||||||
canonical CSV beside this script. Plots go to
|
|
||||||
`plots/png/` and `plots/svg/` beside the selected CSV; override with
|
|
||||||
`--output-dir PATH`.
|
|
||||||
Single-variant comparison CSVs are not accepted.
|
|
||||||
|
|
||||||
The checked-in CSV is an intentional result dataset; plotting uses its current
|
|
||||||
contents without running benchmarks. Invalid reciprocal percentage pairs render
|
|
||||||
as `N/A`; individual legitimate extreme changes are preserved. See
|
|
||||||
[performance availability](../../../../docs/evaluation/methodology.md#functional-validity-and-performance-availability).
|
|
||||||
|
|
||||||
Plots separate modes and pipelines. Throughput-mode latency means output
|
|
||||||
interval. Failed validation is labeled `INVALID`, other non-PASS statuses keep
|
|
||||||
their names, and missing measurements show `N/A`. See the
|
|
||||||
[comparison metric definitions](../../../../docs/evaluation/methodology.md#metric-interpretation).
|
[comparison metric definitions](../../../../docs/evaluation/methodology.md#metric-interpretation).
|
||||||
|
|||||||
@@ -1,156 +0,0 @@
|
|||||||
#!/usr/bin/env python3
|
|
||||||
"""Plot percentage changes from an experiment's combined results_ablation.csv."""
|
|
||||||
|
|
||||||
from __future__ import annotations
|
|
||||||
|
|
||||||
import argparse
|
|
||||||
import csv
|
|
||||||
import math
|
|
||||||
import sys
|
|
||||||
import warnings
|
|
||||||
from dataclasses import dataclass
|
|
||||||
from pathlib import Path
|
|
||||||
|
|
||||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
|
||||||
from plot_style import COLORS, configure_style, save_figure, value_label
|
|
||||||
|
|
||||||
import matplotlib.pyplot as plt
|
|
||||||
from matplotlib.patches import Patch
|
|
||||||
from matplotlib.ticker import PercentFormatter
|
|
||||||
|
|
||||||
from run_ablation import ABLATION_DEFAULT_MODELS, DEFAULT_RESULTS_PATH, RESULT_FIELDS, VARIANTS
|
|
||||||
|
|
||||||
|
|
||||||
VARIANT_COLORS = {variant.name: color for variant, color in zip(VARIANTS, COLORS)}
|
|
||||||
STATUSES = ("PASS", "FAIL", "SKIPPED", "NA")
|
|
||||||
|
|
||||||
|
|
||||||
@dataclass(frozen=True)
|
|
||||||
class Result:
|
|
||||||
variant: str
|
|
||||||
arch: str
|
|
||||||
model: str
|
|
||||||
mode: str
|
|
||||||
pipeline: int
|
|
||||||
validation: str
|
|
||||||
metrics: dict[str, float]
|
|
||||||
|
|
||||||
|
|
||||||
def percentage(value: str) -> float:
|
|
||||||
if value == "NA":
|
|
||||||
return math.nan
|
|
||||||
if not value.endswith("%"):
|
|
||||||
raise ValueError(f"Expected a percentage or NA, got {value!r}")
|
|
||||||
parsed = float(value[:-1])
|
|
||||||
if not math.isfinite(parsed):
|
|
||||||
raise ValueError(f"Non-finite percentage: {value!r}")
|
|
||||||
return parsed
|
|
||||||
|
|
||||||
|
|
||||||
def load_results(path: Path) -> list[Result]:
|
|
||||||
rows = []
|
|
||||||
seen = set()
|
|
||||||
invalid_pairs = []
|
|
||||||
with path.open(newline="", encoding="utf-8-sig") as stream:
|
|
||||||
reader = csv.DictReader(stream)
|
|
||||||
if not {"variant", *RESULT_FIELDS} <= set(reader.fieldnames or []):
|
|
||||||
raise ValueError("Expected the combined ablation CSV with variant and percentage columns; "
|
|
||||||
"a per-variant results_comparison.csv cannot be used here.")
|
|
||||||
for row in reader:
|
|
||||||
variant, mode, status = row["variant"], row["mode"], row["functional_validation"]
|
|
||||||
if variant not in VARIANT_COLORS or mode not in ("latency", "throughput") or status not in STATUSES:
|
|
||||||
raise ValueError(f"Unknown ablation variant, mode, or validation status: {variant}, {mode}, {status}")
|
|
||||||
pipeline = int(row["raptor_pipeline"])
|
|
||||||
key = (variant, row["arch"], row["model"], mode, pipeline)
|
|
||||||
if key in seen:
|
|
||||||
raise ValueError(f"Duplicate ablation configuration: {key}")
|
|
||||||
seen.add(key)
|
|
||||||
metrics = {metric: percentage(row[metric])
|
|
||||||
for metric in ("latency", "throughput", "power", "energy")}
|
|
||||||
# Each may round to -100% alone; reciprocal metrics cannot both do so.
|
|
||||||
if mode == "throughput" and metrics["latency"] == metrics["throughput"] == -100:
|
|
||||||
metrics["latency"] = metrics["throughput"] = math.nan
|
|
||||||
invalid_pairs.append(key)
|
|
||||||
rows.append(Result(variant, row["arch"], row["model"], mode, pipeline, status, metrics))
|
|
||||||
if not rows:
|
|
||||||
raise ValueError("The ablation CSV contains no results")
|
|
||||||
if invalid_pairs:
|
|
||||||
warnings.warn(f"{len(invalid_pairs)} invalid reciprocal latency/throughput percentage pairs; "
|
|
||||||
f"displaying N/A. Examples: {invalid_pairs[:3]}", stacklevel=2)
|
|
||||||
return rows
|
|
||||||
|
|
||||||
|
|
||||||
def draw_metric(results: list[Result], metric: str, ylabel: str, output_dir: Path) -> None:
|
|
||||||
architectures = sorted({row.arch for row in results})
|
|
||||||
present_models = {row.model for row in results}
|
|
||||||
models = [model for model in ABLATION_DEFAULT_MODELS if model in present_models]
|
|
||||||
models += sorted(present_models - set(models))
|
|
||||||
variants = [name for name in VARIANT_COLORS if any(row.variant == name for row in results)]
|
|
||||||
lookup = {(row.arch, row.model, row.variant): row for row in results}
|
|
||||||
values = [row.metrics[metric] for row in results if math.isfinite(row.metrics[metric])]
|
|
||||||
lower, upper = min(0, min(values, default=0)), max(0, max(values, default=0))
|
|
||||||
padding = max(upper - lower, 1) * 0.35
|
|
||||||
fig, axes = plt.subplots(1, len(architectures), sharey=True, squeeze=False,
|
|
||||||
figsize=(7 * len(architectures), 5.2), layout="constrained")
|
|
||||||
handles = [Patch(facecolor=VARIANT_COLORS[name], label=name) for name in variants]
|
|
||||||
if len(values) < len(architectures) * len(models) * len(variants):
|
|
||||||
handles.append(Patch(facecolor="white", edgecolor="#B2BAC5", label="N/A: missing measurement"))
|
|
||||||
fig.legend(handles=handles, loc="outside upper center", ncol=4, frameon=False, fontsize=9.5)
|
|
||||||
width = 0.8 / len(variants)
|
|
||||||
mode, pipeline = results[0].mode, results[0].pipeline
|
|
||||||
for ax, arch in zip(axes[0], architectures):
|
|
||||||
ax.set_axisbelow(True)
|
|
||||||
ax.set_ylim(lower - padding if lower < 0 else 0, upper + padding)
|
|
||||||
ax.set_xlim(-0.5, len(models) - 0.5)
|
|
||||||
ax.set_xticks(range(len(models)), [model.upper() for model in models], fontsize=9.5)
|
|
||||||
ax.set_xlabel(f"{arch.replace('arch-', 'ARCHITECTURE ').upper()}\n"
|
|
||||||
f"{mode.capitalize()}, {pipeline} stage" + ("s" if pipeline != 1 else ""),
|
|
||||||
fontsize=11, weight="bold", labelpad=8)
|
|
||||||
ax.set_ylabel(ylabel, fontsize=10.5)
|
|
||||||
ax.yaxis.set_major_formatter(PercentFormatter(xmax=100, decimals=0))
|
|
||||||
ax.tick_params(axis="both", length=0, pad=6, labelleft=True)
|
|
||||||
ax.grid(axis="y", color="#E6EAF0", linewidth=0.8)
|
|
||||||
ax.axhline(0, color="#536478", linewidth=1.2, linestyle="--")
|
|
||||||
for index, model in enumerate(models):
|
|
||||||
if index % 2 == 0:
|
|
||||||
ax.axvspan(index - 0.5, index + 0.5, facecolor="#F7F9FC", zorder=0)
|
|
||||||
for offset, variant in enumerate(variants):
|
|
||||||
x = index + (offset - (len(variants) - 1) / 2) * width
|
|
||||||
row = lookup.get((arch, model, variant))
|
|
||||||
value = row.metrics[metric] if row else math.nan
|
|
||||||
if not math.isfinite(value):
|
|
||||||
ax.text(x, padding * 0.05, "N/A", rotation=90, ha="center", va="bottom",
|
|
||||||
fontsize=8, color="#85909F")
|
|
||||||
continue
|
|
||||||
ax.bar(x, value, width=width * 0.9, color=VARIANT_COLORS[variant],
|
|
||||||
edgecolor="none", linewidth=0, zorder=2)
|
|
||||||
direction = -1 if value < 0 else 1
|
|
||||||
value_label(ax, x, value + direction * padding * 0.05, f"{value:+,.2f}%",
|
|
||||||
row.validation, direction)
|
|
||||||
save_figure(fig, f"{mode}_pipeline{pipeline}_{metric}", output_dir)
|
|
||||||
|
|
||||||
|
|
||||||
def main() -> None:
|
|
||||||
parser = argparse.ArgumentParser(description=__doc__)
|
|
||||||
parser.add_argument("--input", type=Path, default=DEFAULT_RESULTS_PATH)
|
|
||||||
parser.add_argument("--output-dir", type=Path)
|
|
||||||
args = parser.parse_args()
|
|
||||||
if args.output_dir is None:
|
|
||||||
args.output_dir = args.input.parent / "plots"
|
|
||||||
configure_style()
|
|
||||||
results = load_results(args.input)
|
|
||||||
print(f"Results: {len(results)} configurations, "
|
|
||||||
f"{sum(row.validation != 'PASS' for row in results)} without PASS validation")
|
|
||||||
for mode, pipeline in sorted({(row.mode, row.pipeline) for row in results}):
|
|
||||||
rows = [row for row in results if row.mode == mode and row.pipeline == pipeline]
|
|
||||||
for metric, label in (
|
|
||||||
("latency", "Single-inference latency" if mode == "latency" else "Output interval"),
|
|
||||||
("throughput", "Throughput"), ("power", "Average power"), ("energy", "Energy per inference"),
|
|
||||||
):
|
|
||||||
if metric == "throughput" and mode == "latency":
|
|
||||||
continue
|
|
||||||
draw_metric(rows, metric, f"{label} change from baseline (%)", args.output_dir)
|
|
||||||
|
|
||||||
|
|
||||||
if __name__ == "__main__":
|
|
||||||
main()
|
|
||||||
@@ -72,7 +72,7 @@ The comparator defaults to absolute/relative tolerances `1e-3`/`1e-4`;
|
|||||||
|
|
||||||
Requires Matplotlib from the root `requirements.txt`. Defaults to the supplied
|
Requires Matplotlib from the root `requirements.txt`. Defaults to the supplied
|
||||||
`results_comparison.csv` beside the script and writes `plots/png/` and
|
`results_comparison.csv` beside the script and writes `plots/png/` and
|
||||||
`plots/svg/` there. Use `--input PATH` for a runner-generated CSV and
|
`plots/svg/` and `plots/pdf/` there. Use `--input PATH` for a runner-generated CSV and
|
||||||
`--output-dir PATH` to override the destination. The plotter requires the full
|
`--output-dir PATH` to override the destination. The plotter requires the full
|
||||||
40-case matrix: five models × Arch-A/B × latency pipeline 1 and throughput
|
40-case matrix: five models × Arch-A/B × latency pipeline 1 and throughput
|
||||||
pipelines 2/4/8. Duplicate, missing, unexpected, or inconsistent shared Pimcomp
|
pipelines 2/4/8. Duplicate, missing, unexpected, or inconsistent shared Pimcomp
|
||||||
@@ -81,6 +81,9 @@ configurations are rejected before plots are written.
|
|||||||
| Plot family | Meaning |
|
| Plot family | Meaning |
|
||||||
|---|---|
|
|---|---|
|
||||||
| `validation_status` | Recorded functional status for each compiler and experiment. |
|
| `validation_status` | Recorded functional status for each compiler and experiment. |
|
||||||
|
| `validation_status_compact` | Compact functional status matrix. |
|
||||||
|
| `speedup_compact_*` | Touching single-inference latency and best-pipeline throughput bars for each network. |
|
||||||
|
| `energy_compact_*`, `power_compact_*` | Energy and power advantages for the same cases selected by the compact speedup plots. |
|
||||||
| `latency_*` | Single-inference latency, latency mode only. |
|
| `latency_*` | Single-inference latency, latency mode only. |
|
||||||
| `throughput_*` | Inferences/s, throughput mode only. |
|
| `throughput_*` | Inferences/s, throughput mode only. |
|
||||||
| `longest_stage_latency_*` | Combined single-inference latency and throughput output interval; the latter is a stage-latency proxy. |
|
| `longest_stage_latency_*` | Combined single-inference latency and throughput output interval; the latter is a stage-latency proxy. |
|
||||||
@@ -93,7 +96,28 @@ advantages in either direction. Ratios above 1 favor Raptor: Pimcomp/Raptor for
|
|||||||
latency, power, and energy; Raptor/Pimcomp for throughput. Diverging bars use
|
latency, power, and energy; Raptor/Pimcomp for throughput. Diverging bars use
|
||||||
winning factor minus one, with parity at zero.
|
winning factor minus one, with parity at zero.
|
||||||
|
|
||||||
Failed validation is labeled `INVALID`; other non-PASS statuses keep their
|
The shared `plot_style.py` defines the manuscript palette and display names.
|
||||||
|
Raptor uses sage green; PIMCOMP uses plain sand. Stage patterns,
|
||||||
|
outlines, marker shapes, and status text preserve distinctions in grayscale.
|
||||||
|
Compact plots use blue for the single-inference case and hatched sage
|
||||||
|
green for the throughput case.
|
||||||
|
Each architecture/network pair uses the highest recorded throughput across pipelines.
|
||||||
|
Energy and power use those same cases, without selecting a different pipeline for each metric.
|
||||||
|
Validation labels remain visible for the selected cases. Units appear in axis
|
||||||
|
titles and bar values; axis tick labels contain numbers only.
|
||||||
|
|
||||||
|
Throughput-stage bars touch within each network in both absolute and advantage
|
||||||
|
plots. Bar-label widths are measured before export; dense figures expand only
|
||||||
|
as needed to contain values and invalidity annotations. Legends use the available
|
||||||
|
width and balance entries across rows when wrapping is necessary.
|
||||||
|
|
||||||
|
Comparison panels share y-axis titles and tick labels. Each network in the
|
||||||
|
pipeline plot retains its own scale, with y-axis ticks on both architecture rows.
|
||||||
|
Only the bottom row shows the shared x-axis ticks. Network names appear above
|
||||||
|
the top row; architecture names appear on the right. That plot
|
||||||
|
trims unused vertical range and is intended to span both manuscript columns.
|
||||||
|
|
||||||
|
Failed validation is labeled `P. INVALID`; other non-PASS statuses keep their
|
||||||
names, and missing measurements show `N/A`. Relative bars require both
|
names, and missing measurements show `N/A`. Relative bars require both
|
||||||
compilers to pass to be valid. Repeated Pimcomp batch measurements and statuses
|
compilers to pass to be valid. Repeated Pimcomp batch measurements and statuses
|
||||||
must agree before being collapsed. `FAIL` alone does not identify the cause.
|
must agree before being collapsed. `FAIL` alone does not identify the cause.
|
||||||
|
|||||||
@@ -1,25 +1,25 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
"""Generate publication-ready Raptor vs Pimcomp comparison charts.
|
"""Generate publication-ready Raptor vs PIMCOMP comparison charts.
|
||||||
|
|
||||||
Usage:
|
Usage:
|
||||||
.venv/bin/python validation/tools/pim/pimcomp/compare/generate_plots.py
|
.venv/bin/python validation/tools/pim/pimcomp/compare/generate_plots.py
|
||||||
|
|
||||||
Dependencies: matplotlib (no pandas or seaborn required).
|
Dependencies: matplotlib (no pandas or seaborn required).
|
||||||
By default, reads results_comparison.csv beside this script and writes plots/
|
By default, reads results_comparison.csv beside this script and writes plots/
|
||||||
beside it, with outputs grouped in png/ and svg/ under the output directory.
|
beside it, with outputs grouped in png/, svg/, and pdf/ under the output directory.
|
||||||
|
|
||||||
YOLO models without Pimcomp measurements get separate Raptor-only charts.
|
YOLO models without PIMCOMP measurements get separate Raptor-only charts.
|
||||||
Comparisons use vertical bars for ratios, diverging advantages, and absolute
|
Comparisons use vertical bars for ratios, diverging advantages, and absolute
|
||||||
values, with compiler colors and patterns for the Raptor pipeline stages.
|
values, with compiler colors and patterns for the Raptor pipeline stages.
|
||||||
Throughput-mode latency is the output interval, a bottleneck-stage proxy,
|
Throughput-mode latency is the output interval, a bottleneck-stage proxy,
|
||||||
not end-to-end inference latency. Separate throughput plots use inferences/s.
|
not end-to-end inference latency. Separate throughput plots use inferences/s.
|
||||||
Energy is reported per inference; power is averaged over the simulation.
|
Energy is reported per inference; power is averaged over the simulation.
|
||||||
Numerical results whose compiler did not pass functional validation have a red
|
Numerical results whose compiler did not pass functional validation have a red
|
||||||
status following their value label; missing Pimcomp
|
status following their value label; missing PIMCOMP
|
||||||
measurements appear as 'N/A'. The fixed compiler palette is shared by
|
measurements appear as 'N/A'. The fixed compiler palette is shared by
|
||||||
all metrics. Relative plots use Pimcomp / Raptor for latency, power and energy,
|
all metrics. Relative plots use PIMCOMP / Raptor for latency, power and energy,
|
||||||
and Raptor / Pimcomp for throughput, so >1 favors Raptor. Diverging plots put the winner's
|
and Raptor / PIMCOMP for throughput, so >1 favors Raptor. Diverging plots put the winner's
|
||||||
factor above (Raptor) or below (Pimcomp) parity, with 1x at the zero line.
|
factor above (Raptor) or below (PIMCOMP) parity, with 1x at the zero line.
|
||||||
"""
|
"""
|
||||||
from __future__ import annotations
|
from __future__ import annotations
|
||||||
|
|
||||||
@@ -39,20 +39,22 @@ from matplotlib.lines import Line2D
|
|||||||
from matplotlib.ticker import FuncFormatter, MultipleLocator
|
from matplotlib.ticker import FuncFormatter, MultipleLocator
|
||||||
|
|
||||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||||
from plot_style import COLORS, HATCH_COLOR, TEXT_COLOR, configure_style, fit_value_labels, save_figure, value_label
|
from plot_style import (ARCH_NAMES, BAND_COLOR, COMPILER_NAMES, COMPILER_STYLES,
|
||||||
|
ERROR_COLOR, LINE_COLOR, MATRIX_TEXT_COLOR, METRIC_STYLES,
|
||||||
|
MODE_NAMES, MODEL_NAMES, NETWORK_STYLES,
|
||||||
|
PALETTE, TEXT_COLOR, add_legend, configure_style, fit_value_labels,
|
||||||
|
fit_bar_widths, fit_x_tick_labels, save_figure, value_label)
|
||||||
|
|
||||||
|
|
||||||
COMPILER_COLORS = {"raptor": COLORS[0], "pimcomp": COLORS[2]}
|
|
||||||
STAGE_PATTERNS = {1: "", 2: "..", 4: "//", 8: "\\\\"}
|
STAGE_PATTERNS = {1: "", 2: "..", 4: "//", 8: "\\\\"}
|
||||||
ARCHITECTURES = ("arch-a", "arch-b")
|
ARCHITECTURES = ("arch-a", "arch-b")
|
||||||
MODELS = ("vgg8", "resnet18", "resnet34", "googlenet", "yolo11n")
|
MODELS = ("vgg8", "resnet18", "resnet34", "googlenet", "yolo11n")
|
||||||
CONFIGURATIONS = (("latency", 1), ("throughput", 2), ("throughput", 4), ("throughput", 8))
|
CONFIGURATIONS = (("latency", 1), ("throughput", 2), ("throughput", 4), ("throughput", 8))
|
||||||
LINE_COLOR = "#A8B2C0"
|
|
||||||
VALIDATION_STYLES = {
|
VALIDATION_STYLES = {
|
||||||
"PASS": ("#168C8B", "white", ""),
|
"PASS": PALETTE["sage"],
|
||||||
"FAIL": ("#D18D32", HATCH_COLOR, ""),
|
"FAIL": PALETTE["red"],
|
||||||
"SKIPPED": ("#E7DFF0", TEXT_COLOR, ""),
|
"SKIPPED": PALETTE["pale_sand"],
|
||||||
"NA": ("#E8EDF1", TEXT_COLOR, ""),
|
"NA": PALETTE["pale_blue"],
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
||||||
@@ -85,6 +87,10 @@ class Result:
|
|||||||
def throughput_speedup(self) -> float:
|
def throughput_speedup(self) -> float:
|
||||||
return quotient(self.raptor_throughput, self.pimcomp_throughput)
|
return quotient(self.raptor_throughput, self.pimcomp_throughput)
|
||||||
|
|
||||||
|
@property
|
||||||
|
def speedup(self) -> float:
|
||||||
|
return self.latency_speedup if self.mode == "latency" else self.throughput_speedup
|
||||||
|
|
||||||
|
|
||||||
def positive_float(text: str | None) -> float:
|
def positive_float(text: str | None) -> float:
|
||||||
try:
|
try:
|
||||||
@@ -129,7 +135,7 @@ def load_results(path: Path) -> list[Result]:
|
|||||||
seen[key] = line
|
seen[key] = line
|
||||||
schedule = row["pimcomp_pipeline"].strip().lower()
|
schedule = row["pimcomp_pipeline"].strip().lower()
|
||||||
if schedule != ("element" if mode == "latency" else "batch"):
|
if schedule != ("element" if mode == "latency" else "batch"):
|
||||||
raise ValueError(f"{key}: incompatible Pimcomp schedule {schedule!r}")
|
raise ValueError(f"{key}: incompatible PIMCOMP schedule {schedule!r}")
|
||||||
rows.append(Result(
|
rows.append(Result(
|
||||||
arch=arch,
|
arch=arch,
|
||||||
model=model,
|
model=model,
|
||||||
@@ -185,20 +191,20 @@ def figure_legend(fig, results: list[Result], field: str,
|
|||||||
for _, compiler, value in available} - {None}
|
for _, compiler, value in available} - {None}
|
||||||
if len(visible_compilers) > 1:
|
if len(visible_compilers) > 1:
|
||||||
for compiler in compilers:
|
for compiler in compilers:
|
||||||
handles.append(Patch(facecolor=COMPILER_COLORS[compiler],
|
fill, border = COMPILER_STYLES[compiler]
|
||||||
label=compiler.capitalize() + ("" if absolute else " advantage")))
|
handles.append(Patch(facecolor=fill, edgecolor=border,
|
||||||
|
label=COMPILER_NAMES[compiler] + ("" if absolute else " advantage")))
|
||||||
handles += [Patch(facecolor="white", edgecolor=TEXT_COLOR, hatch=STAGE_PATTERNS[depth],
|
handles += [Patch(facecolor="white", edgecolor=TEXT_COLOR, hatch=STAGE_PATTERNS[depth],
|
||||||
label=f"{depth} stage" + (" (latency)" if depth == 1 else "s"))
|
label=("1 stage (latency)" if absolute else "Single-inference")
|
||||||
|
if depth == 1 else f"{depth} stages")
|
||||||
for depth in sorted(stages) if stages != {1}]
|
for depth in sorted(stages) if stages != {1}]
|
||||||
styles = []
|
styles = []
|
||||||
if len(available) < len(measurements):
|
if len(available) < len(measurements):
|
||||||
styles.append(Patch(facecolor="white", edgecolor="#B2BAC5",
|
styles.append(Patch(facecolor="white", edgecolor=LINE_COLOR,
|
||||||
label="N/A: missing measurement"))
|
label="N/A: missing measurement"))
|
||||||
handles += styles
|
handles += styles
|
||||||
if handles:
|
if handles:
|
||||||
fig.legend(handles=handles, loc="outside upper center", ncol=(len(handles) + 1) // 2,
|
add_legend(fig, handles)
|
||||||
frameon=False, fontsize=9.5, handlelength=1.7, columnspacing=1.5,
|
|
||||||
handletextpad=0.6, labelspacing=0.4)
|
|
||||||
|
|
||||||
|
|
||||||
def absolute_columns(results: list[Result], raptor_only: bool = False) -> list[tuple[Result, str]]:
|
def absolute_columns(results: list[Result], raptor_only: bool = False) -> list[tuple[Result, str]]:
|
||||||
@@ -218,7 +224,7 @@ def absolute_columns(results: list[Result], raptor_only: bool = False) -> list[t
|
|||||||
for row in throughput
|
for row in throughput
|
||||||
}
|
}
|
||||||
if len(signatures) != 1:
|
if len(signatures) != 1:
|
||||||
raise ValueError(f"{arch}/{model}: inconsistent shared Pimcomp batch results across pipelines")
|
raise ValueError(f"{arch}/{model}: inconsistent shared PIMCOMP batch results across pipelines")
|
||||||
columns.append((throughput[0], "pimcomp"))
|
columns.append((throughput[0], "pimcomp"))
|
||||||
return columns
|
return columns
|
||||||
|
|
||||||
@@ -231,34 +237,50 @@ def label_groups(ax, results: list[Result], positions: list[float] | None = None
|
|||||||
for index, (model, rows) in enumerate(groupby(results, key=lambda row: row.model)):
|
for index, (model, rows) in enumerate(groupby(results, key=lambda row: row.model)):
|
||||||
end = start + len(list(rows))
|
end = start + len(list(rows))
|
||||||
centers.append((positions[start] + positions[end - 1]) / 2)
|
centers.append((positions[start] + positions[end - 1]) / 2)
|
||||||
labels.append(model.upper())
|
labels.append(MODEL_NAMES[model])
|
||||||
left = ax.get_xlim()[0] if start == 0 else (positions[start - 1] + positions[start]) / 2
|
left = ax.get_xlim()[0] if start == 0 else (positions[start - 1] + positions[start]) / 2
|
||||||
right = ax.get_xlim()[1] if end == len(results) else (positions[end - 1] + positions[end]) / 2
|
right = ax.get_xlim()[1] if end == len(results) else (positions[end - 1] + positions[end]) / 2
|
||||||
if index % 2 == 0:
|
if index % 2 == 0:
|
||||||
ax.axvspan(left, right, facecolor="#F7F9FC", zorder=0)
|
ax.axvspan(left, right, facecolor=BAND_COLOR, zorder=0)
|
||||||
if index > 0:
|
if index > 0:
|
||||||
ax.axvline(left, color="#DDE3EB", linewidth=0.9)
|
ax.axvline(left, color=LINE_COLOR, linewidth=0.7)
|
||||||
start = end
|
start = end
|
||||||
ax.set_xticks(centers, labels, fontsize=9.5)
|
ax.set_xticks(centers, labels, fontsize=9.5)
|
||||||
|
|
||||||
|
|
||||||
|
def grouped_positions(results: list[Result]) -> list[float]:
|
||||||
|
positions = []
|
||||||
|
for index, row in enumerate(results):
|
||||||
|
if index == 0:
|
||||||
|
positions.append(0.0)
|
||||||
|
continue
|
||||||
|
previous = results[index - 1]
|
||||||
|
gap = 0.8 if row.model != previous.model else 0.4 if row.mode != previous.mode else 0.0
|
||||||
|
positions.append(positions[-1] + 1.0 + gap)
|
||||||
|
return positions
|
||||||
|
|
||||||
|
|
||||||
def diverging_ratio(ratio: float) -> float:
|
def diverging_ratio(ratio: float) -> float:
|
||||||
return ratio - 1 if ratio >= 1 else 1 - 1 / ratio
|
return ratio - 1 if ratio >= 1 else 1 - 1 / ratio
|
||||||
|
|
||||||
|
|
||||||
def draw_validation(results: list[Result], output_dir: Path) -> None:
|
def draw_validation(results: list[Result], output_dir: Path, compact: bool = False) -> None:
|
||||||
columns = absolute_columns(results)
|
cells = {}
|
||||||
configurations = list(dict.fromkeys((row.mode, row.pipeline if compiler == "raptor" else row.pimcomp_schedule,
|
for row, compiler in absolute_columns(results):
|
||||||
compiler) for row, compiler in columns))
|
stage = row.pipeline if compiler == "raptor" else row.pimcomp_schedule
|
||||||
|
key = (row.arch, row.model, row.mode, None if compact else stage, compiler)
|
||||||
|
cells.setdefault(key, set()).add(getattr(row, f"{compiler}_validation"))
|
||||||
|
configurations = list(dict.fromkeys(key[2:] for key in cells))
|
||||||
positions = []
|
positions = []
|
||||||
for index, (mode, _, _) in enumerate(configurations):
|
for index, (mode, _, _) in enumerate(configurations):
|
||||||
gap = 0.4 if index and mode != configurations[index - 1][0] else 0
|
gap = 0.4 if index and mode != configurations[index - 1][0] else 0
|
||||||
positions.append(positions[-1] + 1 + gap if index else 0)
|
positions.append(positions[-1] + 1 + gap if index else 0)
|
||||||
configuration_x = dict(zip(configurations, positions))
|
configuration_x = dict(zip(configurations, positions))
|
||||||
headers = [f"Raptor\n{stage} stages" if mode == "throughput" and compiler == "raptor"
|
headers = [f"Raptor\n{stage} stages" if mode == "throughput" and compiler == "raptor" and not compact
|
||||||
else "Pimcomp\nbatch" if mode == "throughput" else compiler.capitalize()
|
else "PIMCOMP\nbatch" if mode == "throughput" and compiler == "pimcomp" and not compact
|
||||||
|
else COMPILER_NAMES[compiler]
|
||||||
for mode, stage, compiler in configurations]
|
for mode, stage, compiler in configurations]
|
||||||
fig, axes = plt.subplots(1, 2, figsize=(14, 3.8), layout="constrained")
|
fig, axes = plt.subplots(1, 2, figsize=(8.5 if compact else 12, 3.3), layout="constrained")
|
||||||
for ax, arch in zip(axes, ARCHITECTURES):
|
for ax, arch in zip(axes, ARCHITECTURES):
|
||||||
ax.set_xlim(positions[0] - 0.55, positions[-1] + 0.55)
|
ax.set_xlim(positions[0] - 0.55, positions[-1] + 0.55)
|
||||||
ax.set_ylim(len(MODELS) - 0.5, -0.5)
|
ax.set_ylim(len(MODELS) - 0.5, -0.5)
|
||||||
@@ -266,31 +288,41 @@ def draw_validation(results: list[Result], output_dir: Path) -> None:
|
|||||||
for mode, group in groupby(zip(configurations, positions), key=lambda item: item[0][0]):
|
for mode, group in groupby(zip(configurations, positions), key=lambda item: item[0][0]):
|
||||||
group_positions = [x for _, x in group]
|
group_positions = [x for _, x in group]
|
||||||
centers.append((group_positions[0] + group_positions[-1]) / 2)
|
centers.append((group_positions[0] + group_positions[-1]) / 2)
|
||||||
modes.append(mode.capitalize())
|
modes.append("Latency" if mode == "latency" else MODE_NAMES[mode])
|
||||||
ax.set_xticks(centers, modes, fontsize=10)
|
ax.set_xticks(centers, modes, fontsize=10)
|
||||||
ax.set_xticks(positions, headers, minor=True)
|
ax.set_xticks(positions, headers, minor=True)
|
||||||
ax.tick_params(axis="x", which="minor", top=True, labeltop=True,
|
ax.tick_params(axis="x", which="minor", top=True, labeltop=True,
|
||||||
bottom=False, labelbottom=False, length=0, pad=6, labelsize=9)
|
bottom=False, labelbottom=False, length=0, pad=6, labelsize=9)
|
||||||
ax.set_yticks(range(len(MODELS)), [model.upper() for model in MODELS], fontsize=9)
|
ax.set_yticks(range(len(MODELS)), [MODEL_NAMES[model] for model in MODELS], fontsize=9)
|
||||||
ax.tick_params(axis="both", length=0, pad=6)
|
ax.tick_params(axis="both", length=0, pad=6)
|
||||||
architecture = "ARCHITECTURE A" if arch == "arch-a" else "ARCHITECTURE B"
|
ax.tick_params(axis="y", labelleft=arch == ARCHITECTURES[0])
|
||||||
|
architecture = ARCH_NAMES[arch]
|
||||||
ax.set_xlabel(architecture, fontsize=11, weight="bold", labelpad=8)
|
ax.set_xlabel(architecture, fontsize=11, weight="bold", labelpad=8)
|
||||||
for row, compiler in columns:
|
for (cell_arch, model, mode, stage, compiler), statuses in cells.items():
|
||||||
if row.arch != arch:
|
if cell_arch != arch:
|
||||||
continue
|
continue
|
||||||
key = (row.mode, row.pipeline if compiler == "raptor" else row.pimcomp_schedule, compiler)
|
x, y = configuration_x[(mode, stage, compiler)], MODELS.index(model)
|
||||||
x, y = configuration_x[key], MODELS.index(row.model)
|
status = next(status for status in ("FAIL", "SKIPPED", "NA", "PASS") if status in statuses)
|
||||||
status = getattr(row, f"{compiler}_validation")
|
color, border = VALIDATION_STYLES[status]
|
||||||
color, text_color, _ = VALIDATION_STYLES[status]
|
|
||||||
ax.add_patch(Rectangle((x - 0.5, y - 0.42), 1, 0.84,
|
ax.add_patch(Rectangle((x - 0.5, y - 0.42), 1, 0.84,
|
||||||
facecolor=color, edgecolor="#657081", linewidth=0.5))
|
facecolor=color, edgecolor=border, linewidth=0.7))
|
||||||
ax.text(x, y, "N/A" if status == "NA" else status,
|
ax.text(x, y, "N/A" if status == "NA" else status,
|
||||||
color=text_color, ha="center", va="center", fontsize=9)
|
color=MATRIX_TEXT_COLOR, ha="center", va="center", fontsize=12, weight="normal")
|
||||||
save_figure(fig, "validation_status", output_dir)
|
fit_x_tick_labels(fig)
|
||||||
|
save_figure(fig, "validation_status_compact" if compact else "validation_status", output_dir)
|
||||||
|
|
||||||
|
|
||||||
|
def select_compact_speedups(results: list[Result]) -> list[Result]:
|
||||||
|
selected = []
|
||||||
|
for _, group in groupby(results, key=lambda row: (row.arch, row.model, row.mode)):
|
||||||
|
rows = list(group)
|
||||||
|
selected.append(max(rows, key=lambda row: row.speedup if math.isfinite(row.speedup) else -math.inf))
|
||||||
|
return selected
|
||||||
|
|
||||||
|
|
||||||
def draw_metric(results: list[Result], field: str, ylabel: str,
|
def draw_metric(results: list[Result], field: str, ylabel: str,
|
||||||
basename: str, output_dir: Path, diverging: bool = False) -> None:
|
basename: str, output_dir: Path, diverging: bool = False,
|
||||||
|
compact: bool = False) -> None:
|
||||||
data = [diverging_ratio(getattr(row, field)) if diverging else getattr(row, field)
|
data = [diverging_ratio(getattr(row, field)) if diverging else getattr(row, field)
|
||||||
for row in results]
|
for row in results]
|
||||||
available = [v for v in data if math.isfinite(v)]
|
available = [v for v in data if math.isfinite(v)]
|
||||||
@@ -301,49 +333,68 @@ def draw_metric(results: list[Result], field: str, ylabel: str,
|
|||||||
y_min = math.floor((lower - padding) * 2) / 2 if diverging else 0
|
y_min = math.floor((lower - padding) * 2) / 2 if diverging else 0
|
||||||
y_interval = 1.0 if y_max - y_min > 4.5 else 0.5
|
y_interval = 1.0 if y_max - y_min > 4.5 else 0.5
|
||||||
|
|
||||||
fig, axes = plt.subplots(1, 2, sharey=True, figsize=(14, 5.2),
|
fig, axes = plt.subplots(1, 2, sharey=True, figsize=(8.5 if compact else 12, 4.4),
|
||||||
layout="constrained")
|
layout="constrained")
|
||||||
figure_legend(fig, results, field)
|
if compact:
|
||||||
|
single_metric = "latency" if field == "speedup" else field
|
||||||
|
throughput_suffix = "" if field == "speedup" else f" {field}"
|
||||||
|
handles = [Patch(facecolor=METRIC_STYLES["latency"][0],
|
||||||
|
edgecolor=METRIC_STYLES["latency"][1],
|
||||||
|
label=f"Single-inference {single_metric}"),
|
||||||
|
Patch(facecolor=METRIC_STYLES["throughput"][0],
|
||||||
|
edgecolor=METRIC_STYLES["throughput"][1], hatch="//",
|
||||||
|
label=f"Best-case throughput{throughput_suffix}")]
|
||||||
|
|
||||||
for ax, arch in zip(axes, ARCHITECTURES):
|
for ax, arch in zip(axes, ARCHITECTURES):
|
||||||
selected = [row for row in results if row.arch == arch]
|
selected = [row for row in results if row.arch == arch]
|
||||||
ax.set_xlabel("ARCHITECTURE A" if arch == "arch-a" else "ARCHITECTURE B",
|
positions = [index + index // 2 for index in range(len(selected))] if compact else grouped_positions(selected)
|
||||||
|
ax.set_xlabel(ARCH_NAMES[arch],
|
||||||
fontsize=11, weight="bold", labelpad=8)
|
fontsize=11, weight="bold", labelpad=8)
|
||||||
ax.set_axisbelow(True)
|
ax.set_axisbelow(True)
|
||||||
ax.set_ylim(y_min, y_max)
|
ax.set_ylim(y_min, y_max)
|
||||||
ax.set_xlim(-0.5, len(selected) - 0.5)
|
ax.set_xlim(-0.9, positions[-1] + 0.9 if positions else 0.9)
|
||||||
label_groups(ax, selected)
|
label_groups(ax, selected, positions)
|
||||||
ax.tick_params(axis="y", labelleft=True, pad=5, length=0)
|
ax.tick_params(axis="y", labelleft=arch == ARCHITECTURES[0], pad=5, length=0)
|
||||||
ax.tick_params(axis="x", labelsize=9.5, length=0, pad=6)
|
ax.tick_params(axis="x", labelsize=9.5, length=0, pad=6)
|
||||||
ax.yaxis.set_major_locator(MultipleLocator(y_interval))
|
ax.yaxis.set_major_locator(MultipleLocator(y_interval))
|
||||||
if diverging:
|
if diverging:
|
||||||
ax.yaxis.set_major_formatter(FuncFormatter(lambda value, _: f"{abs(value) + 1:g}×"))
|
ax.yaxis.set_major_formatter(FuncFormatter(lambda value, _: f"{abs(value) + 1:g}"))
|
||||||
ax.grid(axis="y", color="#E6EAF0", linewidth=0.8)
|
ax.grid(axis="y")
|
||||||
ax.axhline(0 if diverging else 1, color="#536478", linewidth=1.2,
|
ax.axhline(0 if diverging else 1, color=TEXT_COLOR, linewidth=1.0,
|
||||||
linestyle="--", zorder=3)
|
linestyle="--", zorder=3)
|
||||||
ax.set_ylabel(ylabel, fontsize=10.5, labelpad=6)
|
if arch == ARCHITECTURES[0]:
|
||||||
|
ax.set_ylabel(ylabel, fontsize=10.5, labelpad=6)
|
||||||
|
|
||||||
for position, row in enumerate(selected):
|
for position, row in zip(positions, selected):
|
||||||
ratio = getattr(row, field)
|
ratio = getattr(row, field)
|
||||||
if not math.isfinite(ratio):
|
if not math.isfinite(ratio):
|
||||||
ax.text(position, (y_max - y_min) * 0.02, "N/A", ha="center", va="bottom",
|
ax.text(position, (y_max - y_min) * 0.02, "N/A", ha="center", va="bottom",
|
||||||
fontsize=9.5, color="#85909F", fontstyle="italic")
|
fontsize=9.5, color=TEXT_COLOR, fontstyle="italic")
|
||||||
continue
|
continue
|
||||||
compiler = advantage_compiler(ratio)
|
compiler = advantage_compiler(ratio)
|
||||||
statuses = [status for status in (row.raptor_validation, row.pimcomp_validation) if status != "PASS"]
|
statuses = [status for status in (row.raptor_validation, row.pimcomp_validation) if status != "PASS"]
|
||||||
status = "FAIL" if "FAIL" in statuses else next(iter(statuses), "PASS")
|
status = "FAIL" if "FAIL" in statuses else next(iter(statuses), "PASS")
|
||||||
pattern = STAGE_PATTERNS[row.pipeline]
|
pattern = STAGE_PATTERNS[row.pipeline]
|
||||||
|
color, border = COMPILER_STYLES[compiler] if compiler else ("white", TEXT_COLOR)
|
||||||
|
if compact:
|
||||||
|
pattern = "//" if row.mode == "throughput" else ""
|
||||||
|
color, border = METRIC_STYLES[row.mode]
|
||||||
measurement = diverging_ratio(ratio) if diverging else ratio
|
measurement = diverging_ratio(ratio) if diverging else ratio
|
||||||
ax.bar(position, measurement, width=0.72,
|
ax.bar(position, measurement, width=1.0,
|
||||||
color=COMPILER_COLORS.get(compiler, "#657081"),
|
color=color,
|
||||||
hatch=pattern,
|
hatch=pattern,
|
||||||
edgecolor=TEXT_COLOR if pattern else "none",
|
edgecolor=border, linewidth=0.7, zorder=2)
|
||||||
linewidth=0.7 if pattern else 0.0, zorder=2)
|
|
||||||
direction = -1 if measurement < 0 else 1
|
direction = -1 if measurement < 0 else 1
|
||||||
factor = max(ratio, 1 / ratio) if diverging else ratio
|
factor = max(ratio, 1 / ratio) if diverging else ratio
|
||||||
value_label(ax, position, measurement + direction * (y_max - y_min) * 0.012,
|
value_label(ax, position, measurement + direction * (y_max - y_min) * 0.012,
|
||||||
f"{factor:.2f}×", status, direction, fontsize=9.1)
|
f"{factor:.2f}×", status, direction, fontsize=9.1, invalid_label="P. INVALID")
|
||||||
|
|
||||||
|
fit_bar_widths(fig)
|
||||||
|
if compact:
|
||||||
|
add_legend(fig, handles)
|
||||||
|
else:
|
||||||
|
figure_legend(fig, results, field)
|
||||||
|
fit_x_tick_labels(fig)
|
||||||
fit_value_labels(fig)
|
fit_value_labels(fig)
|
||||||
save_figure(fig, basename, output_dir)
|
save_figure(fig, basename, output_dir)
|
||||||
|
|
||||||
@@ -356,21 +407,13 @@ def draw_absolute(results: list[Result], field: str, ylabel: str,
|
|||||||
for row, compiler in columns
|
for row, compiler in columns
|
||||||
if math.isfinite(getattr(row, f"{compiler}_{field}"))]
|
if math.isfinite(getattr(row, f"{compiler}_{field}"))]
|
||||||
y_max = max(available, default=1.0) * 1.35
|
y_max = max(available, default=1.0) * 1.35
|
||||||
fig, axes = plt.subplots(1, 2, sharey=True, figsize=(14, 5.2),
|
fig, axes = plt.subplots(1, 2, sharey=True, figsize=(12, 4.4),
|
||||||
layout="constrained")
|
layout="constrained")
|
||||||
figure_legend(fig, results, field, absolute=True, raptor_only=raptor_only)
|
|
||||||
|
|
||||||
for ax, arch in zip(axes, ARCHITECTURES):
|
for ax, arch in zip(axes, ARCHITECTURES):
|
||||||
selected = [(row, compiler) for row, compiler in columns if row.arch == arch]
|
selected = [(row, compiler) for row, compiler in columns if row.arch == arch]
|
||||||
positions = []
|
positions = grouped_positions([row for row, _ in selected])
|
||||||
for index, (row, _) in enumerate(selected):
|
ax.set_xlabel(ARCH_NAMES[arch],
|
||||||
if index == 0:
|
|
||||||
positions.append(0.0)
|
|
||||||
continue
|
|
||||||
previous = selected[index - 1][0]
|
|
||||||
gap = 0.8 if row.model != previous.model else 0.4 if row.mode != previous.mode else 0.0
|
|
||||||
positions.append(positions[-1] + 1.0 + gap)
|
|
||||||
ax.set_xlabel("ARCHITECTURE A" if arch == "arch-a" else "ARCHITECTURE B",
|
|
||||||
fontsize=11, weight="bold", labelpad=8)
|
fontsize=11, weight="bold", labelpad=8)
|
||||||
ax.set_axisbelow(True)
|
ax.set_axisbelow(True)
|
||||||
ax.set_ylim(0, y_max)
|
ax.set_ylim(0, y_max)
|
||||||
@@ -384,14 +427,15 @@ def draw_absolute(results: list[Result], field: str, ylabel: str,
|
|||||||
for (_, mode), group in groupby(enumerate(selected), key=lambda item: (item[1][0].model, item[1][0].mode)):
|
for (_, mode), group in groupby(enumerate(selected), key=lambda item: (item[1][0].model, item[1][0].mode)):
|
||||||
group_positions = [positions[position] for position, _ in group]
|
group_positions = [positions[position] for position, _ in group]
|
||||||
centers.append((group_positions[0] + group_positions[-1]) / 2)
|
centers.append((group_positions[0] + group_positions[-1]) / 2)
|
||||||
labels.append(mode.capitalize())
|
labels.append("Latency" if mode == "latency" else MODE_NAMES[mode])
|
||||||
ax.set_xticks(centers, labels, minor=True)
|
ax.set_xticks(centers, labels, minor=True)
|
||||||
ax.tick_params(axis="x", which="minor", length=0, pad=4, labelsize=8)
|
ax.tick_params(axis="x", which="minor", length=0, pad=4, labelsize=8)
|
||||||
ax.set_ylabel(ylabel, fontsize=10.5)
|
if arch == ARCHITECTURES[0]:
|
||||||
|
ax.set_ylabel(ylabel, fontsize=10.5)
|
||||||
if field == "throughput":
|
if field == "throughput":
|
||||||
ax.ticklabel_format(axis="y", style="plain", useOffset=False)
|
ax.ticklabel_format(axis="y", style="plain", useOffset=False)
|
||||||
ax.grid(axis="y", color="#E6EAF0", linewidth=0.8)
|
ax.grid(axis="y")
|
||||||
ax.tick_params(axis="y", length=0, pad=6)
|
ax.tick_params(axis="y", labelleft=arch == ARCHITECTURES[0], length=0, pad=6)
|
||||||
ax.tick_params(axis="x", which="major", length=0,
|
ax.tick_params(axis="x", which="major", length=0,
|
||||||
pad=22 if {row.mode for row, _ in selected} == {"latency", "throughput"} else 6)
|
pad=22 if {row.mode for row, _ in selected} == {"latency", "throughput"} else 6)
|
||||||
|
|
||||||
@@ -399,19 +443,22 @@ def draw_absolute(results: list[Result], field: str, ylabel: str,
|
|||||||
measurement = getattr(row, f"{compiler}_{field}") * scale
|
measurement = getattr(row, f"{compiler}_{field}") * scale
|
||||||
if not math.isfinite(measurement):
|
if not math.isfinite(measurement):
|
||||||
ax.text(x, y_max * 0.02, "N/A", rotation=90, ha="center",
|
ax.text(x, y_max * 0.02, "N/A", rotation=90, ha="center",
|
||||||
va="bottom", fontsize=8, color="#85909F")
|
va="bottom", fontsize=8, color=TEXT_COLOR)
|
||||||
continue
|
continue
|
||||||
status = getattr(row, f"{compiler}_validation")
|
status = getattr(row, f"{compiler}_validation")
|
||||||
pattern = STAGE_PATTERNS[row.pipeline] if compiler == "raptor" else ""
|
pattern = STAGE_PATTERNS[row.pipeline] if compiler == "raptor" else ""
|
||||||
|
fill, border = COMPILER_STYLES[compiler]
|
||||||
ax.bar(x, measurement, width=1.0,
|
ax.bar(x, measurement, width=1.0,
|
||||||
color=COMPILER_COLORS[compiler],
|
color=fill,
|
||||||
hatch=pattern,
|
hatch=pattern,
|
||||||
edgecolor=TEXT_COLOR if pattern else "none",
|
edgecolor=border, linewidth=0.7, zorder=2)
|
||||||
linewidth=0.7 if pattern else 0.0, zorder=2)
|
|
||||||
label = (f"{measurement:,.2f}".rstrip("0").rstrip(".")
|
label = (f"{measurement:,.2f}".rstrip("0").rstrip(".")
|
||||||
if field == "throughput" else f"{measurement:.3g}")
|
if field == "throughput" else f"{measurement:.3g}")
|
||||||
value_label(ax, x, measurement + y_max * 0.012, label, status)
|
value_label(ax, x, measurement + y_max * 0.012, label, status, invalid_label="P. INVALID")
|
||||||
|
|
||||||
|
fit_bar_widths(fig)
|
||||||
|
figure_legend(fig, results, field, absolute=True, raptor_only=raptor_only)
|
||||||
|
fit_x_tick_labels(fig)
|
||||||
fit_value_labels(fig)
|
fit_value_labels(fig)
|
||||||
save_figure(fig, basename, output_dir)
|
save_figure(fig, basename, output_dir)
|
||||||
|
|
||||||
@@ -419,43 +466,50 @@ def draw_absolute(results: list[Result], field: str, ylabel: str,
|
|||||||
def draw_pipeline_effect(results: list[Result], output_dir: Path) -> None:
|
def draw_pipeline_effect(results: list[Result], output_dir: Path) -> None:
|
||||||
rows = [row for row in results if row.mode == "throughput" or row.pipeline == 1]
|
rows = [row for row in results if row.mode == "throughput" or row.pipeline == 1]
|
||||||
stages = sorted({row.pipeline for row in rows})
|
stages = sorted({row.pipeline for row in rows})
|
||||||
panels = [(model, color) for model, color in zip(MODELS, COLORS)
|
panels = [model for model in MODELS if any(row.model == model for row in rows)]
|
||||||
if any(row.model == model for row in rows)]
|
|
||||||
lookup = {(row.arch, row.model, row.pipeline): row for row in rows}
|
lookup = {(row.arch, row.model, row.pipeline): row for row in rows}
|
||||||
rates = {key: quotient(1000, row.raptor_latency) if row.mode == "latency" else row.raptor_throughput
|
rates = {key: quotient(1000, row.raptor_latency) if row.mode == "latency" else row.raptor_throughput
|
||||||
for key, row in lookup.items()}
|
for key, row in lookup.items()}
|
||||||
fig, axes = plt.subplots(len(ARCHITECTURES), len(panels), sharex=True, sharey="col",
|
fig, axes = plt.subplots(len(ARCHITECTURES), len(panels), sharex=True, sharey="col",
|
||||||
figsize=(3 * len(panels), 7.2), squeeze=False, layout="constrained")
|
figsize=(2.6 * len(panels), 4.3), squeeze=False, layout="constrained")
|
||||||
for column, (model, color) in enumerate(panels):
|
fig.supylabel("Throughput (inferences/s)", fontsize=10.5)
|
||||||
|
for column, model in enumerate(panels):
|
||||||
|
fill, border = NETWORK_STYLES[model]
|
||||||
for index, arch in enumerate(ARCHITECTURES):
|
for index, arch in enumerate(ARCHITECTURES):
|
||||||
ax = axes[index, column]
|
ax = axes[index, column]
|
||||||
keys = [(arch, model, stage) for stage in stages]
|
keys = [(arch, model, stage) for stage in stages]
|
||||||
values = [rates.get(key, math.nan) for key in keys]
|
values = [rates.get(key, math.nan) for key in keys]
|
||||||
valid = [key in lookup and lookup[key].raptor_validation == "PASS" for key in keys]
|
valid = [key in lookup and lookup[key].raptor_validation == "PASS" for key in keys]
|
||||||
ax.plot(stages, values, color=color, linewidth=1.8)
|
ax.plot(stages, values, color=border, linewidth=1.8)
|
||||||
for stage, value, passed in zip(stages, values, valid):
|
for stage, value, passed in zip(stages, values, valid):
|
||||||
if math.isfinite(value):
|
if math.isfinite(value):
|
||||||
marker = ("s" if stage == 1 else "o") if passed else "x"
|
marker = ("s" if stage == 1 else "o") if passed else "x"
|
||||||
ax.plot(stage, value, color=color, marker=marker,
|
status = lookup[(arch, model, stage)].raptor_validation
|
||||||
|
ax.plot(stage, value, color=ERROR_COLOR if status == "FAIL" else border,
|
||||||
|
markerfacecolor=fill, marker=marker,
|
||||||
markersize=6, linestyle="none")
|
markersize=6, linestyle="none")
|
||||||
ax.set_xticks(stages)
|
ax.set_xticks(stages)
|
||||||
ax.set_xlabel(model.upper(), fontsize=10.5, weight="bold", labelpad=8)
|
if index == 0:
|
||||||
ax.grid(axis="y", color="#E6EAF0", linewidth=0.8)
|
ax.set_title(MODEL_NAMES[model], fontsize=10.5, weight="normal", pad=8)
|
||||||
ax.tick_params(axis="both", length=0, pad=6, labelbottom=True, labelleft=True)
|
ax.grid(axis="y")
|
||||||
|
ax.tick_params(axis="both", length=0, pad=6,
|
||||||
|
labelbottom=index == len(ARCHITECTURES) - 1, labelleft=True)
|
||||||
ax.ticklabel_format(axis="y", style="plain", useOffset=False)
|
ax.ticklabel_format(axis="y", style="plain", useOffset=False)
|
||||||
if column == 0:
|
if column == len(panels) - 1:
|
||||||
ax.set_ylabel(f"{arch.replace('arch-', 'ARCHITECTURE ').upper()}\n"
|
ax.yaxis.set_label_position("right")
|
||||||
"Throughput (inferences/s)", fontsize=10.5)
|
ax.set_ylabel(ARCH_NAMES[arch], fontsize=10.5, weight="bold", rotation=270, labelpad=24)
|
||||||
for ax in axes.flat:
|
for ax in axes.flat:
|
||||||
ax.set_ylim(bottom=0)
|
ax.margins(y=0.08)
|
||||||
handles = [Line2D([], [], color="#657081", marker=marker, linestyle="none", label=label)
|
handles = [Line2D([], [], color=TEXT_COLOR, markerfacecolor="white",
|
||||||
for marker, label in (("s", "Derived as 1000ms / latency_ms"),
|
marker=marker, linestyle="none", label=label)
|
||||||
|
for marker, label in (("s", "Derived as 1 / latency"),
|
||||||
("o", "Throughput measurement"))]
|
("o", "Throughput measurement"))]
|
||||||
if any(row.raptor_validation != "PASS" and math.isfinite(rates[key]) for key, row in lookup.items()):
|
if any(row.raptor_validation != "PASS" and math.isfinite(rates[key]) for key, row in lookup.items()):
|
||||||
handles.append(Line2D([], [], color="#424B58", marker="x", linestyle="none",
|
handles.append(Line2D([], [], color=TEXT_COLOR, marker="x", linestyle="none",
|
||||||
label="Raptor validation not PASS"))
|
label="Raptor validation not PASS"))
|
||||||
fig.legend(handles=handles, loc="outside upper center", ncol=len(handles), frameon=False, fontsize=9.5)
|
add_legend(fig, handles)
|
||||||
fig.supxlabel("Raptor pipeline stages", fontsize=11)
|
fig.supxlabel("Raptor pipeline stages", fontsize=11)
|
||||||
|
fit_x_tick_labels(fig)
|
||||||
save_figure(fig, "raptor_pipeline_throughput", output_dir)
|
save_figure(fig, "raptor_pipeline_throughput", output_dir)
|
||||||
|
|
||||||
|
|
||||||
@@ -471,6 +525,7 @@ def main() -> None:
|
|||||||
print(f"Results: {len(results)} configurations, {valid} valid comparisons, "
|
print(f"Results: {len(results)} configurations, {valid} valid comparisons, "
|
||||||
f"{len(results)-valid} comparisons without two PASS statuses")
|
f"{len(results)-valid} comparisons without two PASS statuses")
|
||||||
draw_validation(results, args.output_dir)
|
draw_validation(results, args.output_dir)
|
||||||
|
draw_validation(results, args.output_dir, compact=True)
|
||||||
draw_pipeline_effect(results, args.output_dir)
|
draw_pipeline_effect(results, args.output_dir)
|
||||||
yolo_only_models = {
|
yolo_only_models = {
|
||||||
model for model in MODELS if model.startswith("yolo")
|
model for model in MODELS if model.startswith("yolo")
|
||||||
@@ -480,12 +535,19 @@ def main() -> None:
|
|||||||
}
|
}
|
||||||
comparisons = [row for row in results if row.model not in yolo_only_models]
|
comparisons = [row for row in results if row.model not in yolo_only_models]
|
||||||
yolo_only = [row for row in results if row.model in yolo_only_models]
|
yolo_only = [row for row in results if row.model in yolo_only_models]
|
||||||
|
compact_speedups = select_compact_speedups(comparisons)
|
||||||
|
for metric, label in (("speedup", "Speedup (x)"),
|
||||||
|
("energy", "Energy-per-inference advantage (x)"),
|
||||||
|
("power", "Average power advantage (x)")):
|
||||||
|
for suffix, diverging in (("comparison", False), ("diverging", True)):
|
||||||
|
draw_metric(compact_speedups, metric, label,
|
||||||
|
f"{metric}_compact_{suffix}", args.output_dir, diverging=diverging, compact=True)
|
||||||
for metric, name, label, mode in [
|
for metric, name, label, mode in [
|
||||||
("latency_speedup", "longest_stage_latency", "Longest stage latency speedup (x)", None),
|
("latency_speedup", "longest_stage_latency", "Longest stage latency speedup (x)", None),
|
||||||
("latency_speedup", "latency", "Single-inference latency speedup (x)", "latency"),
|
("latency_speedup", "latency", "Single-inference latency speedup (x)", "latency"),
|
||||||
("throughput_speedup", "throughput", "Throughput speedup (x)", "throughput"),
|
("throughput_speedup", "throughput", "Throughput speedup (x)", "throughput"),
|
||||||
("power", "power", "Average power advantage (x)", None),
|
("power", "power", "Average power advantage (x)", None),
|
||||||
("energy", "energy", "Energy per inference advantage (x)", None),
|
("energy", "energy", "Energy-per-inference advantage (x)", None),
|
||||||
]:
|
]:
|
||||||
selected = [row for row in comparisons if mode is None or row.mode == mode]
|
selected = [row for row in comparisons if mode is None or row.mode == mode]
|
||||||
draw_metric(selected, metric, label,
|
draw_metric(selected, metric, label,
|
||||||
|
|||||||
@@ -1,4 +1,4 @@
|
|||||||
"""Shared appearance and PNG/SVG export for PIM result plots."""
|
"""Shared appearance and PNG/SVG/PDF export for PIM result plots."""
|
||||||
|
|
||||||
from pathlib import Path
|
from pathlib import Path
|
||||||
from math import ceil
|
from math import ceil
|
||||||
@@ -12,9 +12,32 @@ from matplotlib.patheffects import AbstractPathEffect
|
|||||||
from matplotlib.transforms import Affine2D, IdentityTransform, TransformedPath
|
from matplotlib.transforms import Affine2D, IdentityTransform, TransformedPath
|
||||||
|
|
||||||
|
|
||||||
TEXT_COLOR = "#243449"
|
PALETTE = {
|
||||||
HATCH_COLOR = "#C62828"
|
"sage": ("#B3D5CA", "#4B7C6A"),
|
||||||
COLORS = ("#2B65A1", "#168C8B", "#D18D32", "#8062A5", "#B65D60")
|
"sand": ("#E1BC8B", "#856A44"),
|
||||||
|
"pale_blue": ("#EDF3FA", "#456B90"),
|
||||||
|
"blue": ("#B3CCE3", "#456B90"),
|
||||||
|
"pale_green": ("#EAF5F1", "#3E8271"),
|
||||||
|
"pale_sand": ("#FCF1E5", "#A87940"),
|
||||||
|
"red": ("#FF9494", "#C52020"),
|
||||||
|
"purple": ("#D1C0E0", "#77558F"),
|
||||||
|
}
|
||||||
|
TEXT_COLOR = "#222222"
|
||||||
|
MATRIX_TEXT_COLOR = "#1D2530"
|
||||||
|
ERROR_COLOR = PALETTE["red"][1]
|
||||||
|
GRID_COLOR = "#E6E9EC"
|
||||||
|
LINE_COLOR = "#C9CED3"
|
||||||
|
BAND_COLOR = "#F7F9FA"
|
||||||
|
COMPILER_STYLES = {"raptor": PALETTE["sage"], "pimcomp": PALETTE["sand"]}
|
||||||
|
METRIC_STYLES = {"latency": PALETTE["blue"], "throughput": PALETTE["sage"]}
|
||||||
|
NETWORK_STYLES = {"vgg8": PALETTE["blue"], "resnet18": PALETTE["sage"],
|
||||||
|
"resnet34": PALETTE["sand"], "googlenet": PALETTE["purple"],
|
||||||
|
"yolo11n": PALETTE["red"]}
|
||||||
|
COMPILER_NAMES = {"raptor": "Raptor", "pimcomp": "PIMCOMP"}
|
||||||
|
ARCH_NAMES = {"arch-a": "Arch-A", "arch-b": "Arch-B"}
|
||||||
|
MODEL_NAMES = {"vgg8": "VGG-8", "resnet18": "ResNet-18", "resnet34": "ResNet-34",
|
||||||
|
"googlenet": "GoogLeNet", "yolo11n": "YOLO11n"}
|
||||||
|
MODE_NAMES = {"latency": "Single-inference", "throughput": "Throughput"}
|
||||||
|
|
||||||
|
|
||||||
class CenteredHatch(AbstractPathEffect):
|
class CenteredHatch(AbstractPathEffect):
|
||||||
@@ -77,51 +100,130 @@ def configure_style() -> None:
|
|||||||
"text.color": TEXT_COLOR,
|
"text.color": TEXT_COLOR,
|
||||||
"xtick.color": TEXT_COLOR,
|
"xtick.color": TEXT_COLOR,
|
||||||
"ytick.color": TEXT_COLOR,
|
"ytick.color": TEXT_COLOR,
|
||||||
"axes.edgecolor": "#CFD6DF",
|
"axes.edgecolor": LINE_COLOR,
|
||||||
|
"grid.color": GRID_COLOR,
|
||||||
|
"grid.linewidth": 0.8,
|
||||||
|
"hatch.color": TEXT_COLOR,
|
||||||
"savefig.facecolor": "white",
|
"savefig.facecolor": "white",
|
||||||
"figure.facecolor": "white",
|
"figure.facecolor": "white",
|
||||||
})
|
})
|
||||||
|
|
||||||
|
|
||||||
|
def add_legend(fig, handles) -> None:
|
||||||
|
"""Use the fewest balanced legend rows that fit the figure width."""
|
||||||
|
renderer = fig.canvas.get_renderer()
|
||||||
|
for rows in range(1, len(handles) + 1):
|
||||||
|
columns = ceil(len(handles) / rows)
|
||||||
|
grid = []
|
||||||
|
start = 0
|
||||||
|
for row in range(rows):
|
||||||
|
count = len(handles) // rows + (row < len(handles) % rows)
|
||||||
|
grid.append(handles[start:start + count] + [None] * (columns - count))
|
||||||
|
start += count
|
||||||
|
ordered = [handle for column in zip(*grid) for handle in column]
|
||||||
|
legend = fig.legend(handles=[handle if handle is not None else Patch(visible=False) for handle in ordered],
|
||||||
|
labels=[handle.get_label() if handle is not None else "" for handle in ordered],
|
||||||
|
loc="outside upper center", ncols=columns, frameon=False, fontsize=9.5,
|
||||||
|
handlelength=1.7, columnspacing=1.5, handletextpad=0.6, labelspacing=0.4)
|
||||||
|
for handle, patch, text in zip(ordered, legend.legend_handles, legend.get_texts()):
|
||||||
|
if handle is None:
|
||||||
|
patch.set_visible(False)
|
||||||
|
text.set_visible(False)
|
||||||
|
if legend.get_window_extent(renderer).width <= fig.bbox.width - 20:
|
||||||
|
return
|
||||||
|
legend.remove()
|
||||||
|
|
||||||
|
|
||||||
def value_label(ax, x, y, label: str, status: str = "PASS", direction: int = 1,
|
def value_label(ax, x, y, label: str, status: str = "PASS", direction: int = 1,
|
||||||
fontsize: float = 8) -> None:
|
fontsize: float = 8, invalid_label: str = "INVALID") -> None:
|
||||||
alignment = "bottom" if direction > 0 else "top"
|
alignment = "bottom" if direction > 0 else "top"
|
||||||
rotation = 90 if direction > 0 else -90
|
rotation = 90 if direction > 0 else -90
|
||||||
text = ax.text(x, y, label, rotation=rotation, ha="center", va=alignment,
|
offset = direction * fontsize * 0.65 if status != "PASS" else 0
|
||||||
fontsize=fontsize, color=TEXT_COLOR)
|
ax.annotate(label, (x, y), xytext=(-offset, 0), textcoords="offset points",
|
||||||
|
rotation=rotation, ha="center", va=alignment, fontsize=fontsize,
|
||||||
|
color=TEXT_COLOR)
|
||||||
if status != "PASS":
|
if status != "PASS":
|
||||||
height = text.get_window_extent(ax.figure.canvas.get_renderer()).height
|
ax.annotate(invalid_label if status == "FAIL" else status, (x, y),
|
||||||
offset = direction * (height * 72 / ax.figure.dpi + 3)
|
xytext=(offset, 0), textcoords="offset points",
|
||||||
ax.annotate("INVALID" if status == "FAIL" else status, (x, y),
|
|
||||||
xytext=(0, offset), textcoords="offset points",
|
|
||||||
rotation=rotation, ha="center", va=alignment, fontsize=fontsize,
|
rotation=rotation, ha="center", va=alignment, fontsize=fontsize,
|
||||||
color=HATCH_COLOR)
|
color=ERROR_COLOR if status == "FAIL" else TEXT_COLOR)
|
||||||
|
|
||||||
|
|
||||||
|
def fit_x_tick_labels(fig) -> None:
|
||||||
|
"""Rotate only overlapping labels, clockwise by 30 degrees."""
|
||||||
|
fig.canvas.draw()
|
||||||
|
renderer = fig.canvas.get_renderer()
|
||||||
|
for minor in (False, True):
|
||||||
|
labels = [(text, text.get_window_extent(renderer))
|
||||||
|
for ax in fig.axes for text in ax.get_xticklabels(minor=minor)
|
||||||
|
if text.get_visible() and text.get_text()]
|
||||||
|
crowded = set()
|
||||||
|
for index, (text, bounds) in enumerate(labels):
|
||||||
|
for other_text, other_bounds in labels[:index]:
|
||||||
|
if bounds.overlaps(other_bounds):
|
||||||
|
crowded.update((text, other_text))
|
||||||
|
for text in crowded:
|
||||||
|
text.set_rotation(-30)
|
||||||
|
text.set_rotation_mode("anchor")
|
||||||
|
text.set_ha("right" if text.get_va() == "bottom" else "left")
|
||||||
|
fig.canvas.draw()
|
||||||
|
renderer = fig.canvas.get_renderer()
|
||||||
|
for ax in fig.axes:
|
||||||
|
bottom_minor = [text for text in ax.get_xticklabels(minor=True)
|
||||||
|
if text.get_va() == "top" and text.get_visible() and text.get_text()]
|
||||||
|
if bottom_minor:
|
||||||
|
height = max(text.get_window_extent(renderer).height for text in bottom_minor)
|
||||||
|
ax.tick_params(axis="x", which="major", pad=height * 72 / fig.dpi + 6)
|
||||||
|
fig.canvas.draw()
|
||||||
|
|
||||||
|
|
||||||
|
def fit_bar_widths(fig) -> None:
|
||||||
|
"""Reserve enough horizontal space for each bar's rendered annotations."""
|
||||||
|
fig.canvas.draw()
|
||||||
|
renderer = fig.canvas.get_renderer()
|
||||||
|
scale = 1.0
|
||||||
|
for ax in fig.axes:
|
||||||
|
labels = {}
|
||||||
|
for text in ax.texts:
|
||||||
|
if hasattr(text, "xy"):
|
||||||
|
labels.setdefault(text.xy[0], []).append(text.get_window_extent(renderer))
|
||||||
|
for container in ax.containers:
|
||||||
|
for bar in container.patches:
|
||||||
|
bounds = labels.get(bar.get_x() + bar.get_width() / 2, [])
|
||||||
|
if bounds:
|
||||||
|
width = max(box.x1 for box in bounds) - min(box.x0 for box in bounds)
|
||||||
|
scale = max(scale, (width + renderer.points_to_pixels(2)) / bar.get_window_extent(renderer).width)
|
||||||
|
if scale > 1:
|
||||||
|
fig.set_size_inches(fig.get_figwidth() * scale, fig.get_figheight())
|
||||||
|
|
||||||
|
|
||||||
def fit_value_labels(fig) -> None:
|
def fit_value_labels(fig) -> None:
|
||||||
"""Reserve vertical space for rotated labels using their rendered size."""
|
"""Fit common vertical limits to the data and rendered value labels."""
|
||||||
for ax in fig.axes:
|
for ax in fig.axes:
|
||||||
for text in ax.texts:
|
for text in ax.texts:
|
||||||
text.set_in_layout(False)
|
text.set_in_layout(False)
|
||||||
fig.canvas.draw()
|
fig.canvas.draw()
|
||||||
renderer = fig.canvas.get_renderer()
|
renderer = fig.canvas.get_renderer()
|
||||||
limits = []
|
limits = []
|
||||||
|
labels = []
|
||||||
for ax in fig.axes:
|
for ax in fig.axes:
|
||||||
bounds = ax.get_window_extent(renderer)
|
bounds = ax.get_window_extent(renderer)
|
||||||
labels = []
|
|
||||||
for text in ax.texts:
|
for text in ax.texts:
|
||||||
x, y = text.xy if hasattr(text, "xy") else text.get_position()
|
x, y = text.xy if hasattr(text, "xy") else text.get_position()
|
||||||
anchor = ax.transData.transform((x, y))[1]
|
anchor = ax.transData.transform((x, y))[1]
|
||||||
extent = text.get_window_extent(renderer)
|
extent = text.get_window_extent(renderer)
|
||||||
labels.append((y, max(0, anchor - extent.y0 + 3) / bounds.height,
|
labels.append((y, max(0, anchor - extent.y0 + 3) / bounds.height,
|
||||||
max(0, extent.y1 - anchor + 3) / bounds.height))
|
max(0, extent.y1 - anchor + 3) / bounds.height))
|
||||||
lower, upper = ax.get_ylim()
|
limits.append(ax.dataLim.intervaly if ax.has_data() else ax.get_ylim())
|
||||||
for _ in range(12):
|
lower = min(0, *(lower for lower, _ in limits))
|
||||||
lower = min([lower, *((y - below * upper) / (1 - below) for y, below, _ in labels)])
|
upper = max(0, *(upper for _, upper in limits))
|
||||||
upper = max([upper, *((y - above * lower) / (1 - above) for y, _, above in labels)])
|
if lower == upper:
|
||||||
limits.append((lower, upper))
|
upper = lower + 1
|
||||||
|
for _ in range(12):
|
||||||
|
lower = min([lower, *((y - below * upper) / (1 - below) for y, below, _ in labels)])
|
||||||
|
upper = max([upper, *((y - above * lower) / (1 - above) for y, _, above in labels)])
|
||||||
for ax in fig.axes:
|
for ax in fig.axes:
|
||||||
ax.set_ylim(min(lower for lower, _ in limits), max(upper for _, upper in limits))
|
ax.set_ylim(lower, upper)
|
||||||
|
|
||||||
|
|
||||||
def save_figure(fig, basename: str, output_dir: Path) -> None:
|
def save_figure(fig, basename: str, output_dir: Path) -> None:
|
||||||
@@ -129,7 +231,7 @@ def save_figure(fig, basename: str, output_dir: Path) -> None:
|
|||||||
hatch = patch.get_hatch() or ""
|
hatch = patch.get_hatch() or ""
|
||||||
if any(symbol in hatch for symbol in "./\\"):
|
if any(symbol in hatch for symbol in "./\\"):
|
||||||
patch.set_path_effects([CenteredHatch()])
|
patch.set_path_effects([CenteredHatch()])
|
||||||
for extension in ("png", "svg"):
|
for extension in ("png", "svg", "pdf"):
|
||||||
target = output_dir / extension / f"{basename}.{extension}"
|
target = output_dir / extension / f"{basename}.{extension}"
|
||||||
target.parent.mkdir(parents=True, exist_ok=True)
|
target.parent.mkdir(parents=True, exist_ok=True)
|
||||||
fig.savefig(target, dpi=220, bbox_inches="tight", pad_inches=0.04)
|
fig.savefig(target, dpi=220, bbox_inches="tight", pad_inches=0.04)
|
||||||
|
|||||||
Reference in New Issue
Block a user