This commit is contained in:
@@ -0,0 +1,178 @@
|
||||
Operation,Arch,Result (l),Result (t),Compile (l),Host mem (l),Cores mem (l),Cores (l),Xbars (l),Latency (l),Power (l),Energy (l),Compile (t),Host mem (t),Cores mem (t),Cores (t),Xbars (t),Avg latency (t),Throughput (t),Avg power (t),Avg energy (t)
|
||||
add/after_gemm,arch-a,PASS,PASS,0.040 s,0.01 MiB,0.01 MiB,4,4,SKIP,SKIP,SKIP,0.041 s,0.01 MiB,0.01 MiB,5,4,SKIP,SKIP,SKIP,SKIP
|
||||
add/basic,arch-a,PASS,PASS,0.033 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.034 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
add/broadcast_row,arch-a,PASS,PASS,0.034 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.034 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
add/channel_broadcast_1024,arch-a,PASS,PASS,0.035 s,0.02 MiB,0.01 MiB,1,0,SKIP,SKIP,SKIP,0.035 s,0.02 MiB,0.01 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
add/leading_dimension_broadcast,arch-a,PASS,PASS,0.035 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.035 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
concat/channel_axis,arch-a,PASS,PASS,0.035 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.035 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
concat/negative_axis,arch-a,PASS,PASS,0.033 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.035 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
concat/three_inputs_channel_axis,arch-a,PASS,PASS,0.036 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.035 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
conv/batch_2,arch-a,PASS,PASS,0.042 s,0.00 MiB,0.00 MiB,2,2,SKIP,SKIP,SKIP,0.050 s,0.00 MiB,0.01 MiB,4,2,SKIP,SKIP,SKIP,SKIP
|
||||
conv/batch_4_pointwise,arch-a,PASS,PASS,0.044 s,0.00 MiB,0.01 MiB,4,4,SKIP,SKIP,SKIP,0.052 s,0.01 MiB,0.01 MiB,5,4,SKIP,SKIP,SKIP,SKIP
|
||||
conv/depthwise_1024_channels,arch-a,PASS,PASS,0.068 s,0.19 MiB,0.38 MiB,128,128,SKIP,SKIP,SKIP,0.116 s,0.26 MiB,0.50 MiB,130,128,SKIP,SKIP,SKIP,SKIP
|
||||
conv/depthwise_grouped,arch-a,PASS,PASS,0.039 s,0.01 MiB,0.00 MiB,4,4,SKIP,SKIP,SKIP,0.044 s,0.01 MiB,0.00 MiB,6,4,SKIP,SKIP,SKIP,SKIP
|
||||
conv/dilated_3x3,arch-a,PASS,PASS,0.050 s,0.01 MiB,0.01 MiB,9,9,SKIP,SKIP,SKIP,0.050 s,0.01 MiB,0.01 MiB,10,9,SKIP,SKIP,SKIP,SKIP
|
||||
conv/dynamic,arch-a,PASS,PASS,0.039 s,0.00 MiB,0.00 MiB,5,0,SKIP,SKIP,SKIP,0.045 s,0.00 MiB,0.00 MiB,5,0,SKIP,SKIP,SKIP,SKIP
|
||||
conv/explicit_padding,arch-a,PASS,PASS,0.042 s,0.01 MiB,0.02 MiB,16,16,SKIP,SKIP,SKIP,0.055 s,0.01 MiB,0.02 MiB,17,16,SKIP,SKIP,SKIP,SKIP
|
||||
conv/grouped_many_groups,arch-a,PASS,PASS,0.547 s,0.05 MiB,0.09 MiB,64,64,SKIP,SKIP,SKIP,0.628 s,0.05 MiB,0.14 MiB,124,64,SKIP,SKIP,SKIP,SKIP
|
||||
conv/grouped_two_groups,arch-a,PASS,PASS,0.049 s,0.00 MiB,0.00 MiB,2,2,SKIP,SKIP,SKIP,0.053 s,0.00 MiB,0.00 MiB,4,2,SKIP,SKIP,SKIP,SKIP
|
||||
conv/huge_pointwise_1024,arch-a,PASS,PASS,0.160 s,0.01 MiB,0.09 MiB,64,64,SKIP,SKIP,SKIP,0.190 s,0.02 MiB,0.11 MiB,74,64,SKIP,SKIP,SKIP,SKIP
|
||||
conv/huge_pointwise_1024_dynamic,arch-a,PASS,PASS,0.073 s,8.04 MiB,12.55 MiB,167,0,SKIP,SKIP,SKIP,0.249 s,8.05 MiB,12.45 MiB,165,0,SKIP,SKIP,SKIP,SKIP
|
||||
conv/input_224_7x7_stride2,arch-a,PASS,PASS,0.777 s,24.14 MiB,61.87 MiB,168,169,SKIP,SKIP,SKIP,3.669 s,31.18 MiB,68.93 MiB,165,169,SKIP,SKIP,SKIP,SKIP
|
||||
conv/kernel_2x2,arch-a,PASS,PASS,0.041 s,0.00 MiB,0.00 MiB,1,1,SKIP,SKIP,SKIP,0.063 s,0.00 MiB,0.00 MiB,3,1,SKIP,SKIP,SKIP,SKIP
|
||||
conv/kernel_3x3,arch-a,PASS,PASS,0.043 s,0.01 MiB,0.01 MiB,9,9,SKIP,SKIP,SKIP,0.046 s,0.01 MiB,0.01 MiB,11,9,SKIP,SKIP,SKIP,SKIP
|
||||
conv/kernel_equals_input_spatial,arch-a,PASS,PASS,0.042 s,0.00 MiB,0.00 MiB,2,2,SKIP,SKIP,SKIP,0.048 s,0.00 MiB,0.00 MiB,4,2,SKIP,SKIP,SKIP,SKIP
|
||||
conv/large_input_channels_1x1,arch-a,PASS,PASS,0.079 s,0.01 MiB,0.01 MiB,8,8,SKIP,SKIP,SKIP,0.081 s,0.01 MiB,0.02 MiB,10,8,SKIP,SKIP,SKIP,SKIP
|
||||
conv/large_output_channels_1x1,arch-a,PASS,PASS,0.081 s,0.00 MiB,0.01 MiB,8,8,SKIP,SKIP,SKIP,0.087 s,0.01 MiB,0.02 MiB,9,8,SKIP,SKIP,SKIP,SKIP
|
||||
conv/large_spatial,arch-a,PASS,PASS,0.048 s,0.01 MiB,0.04 MiB,36,36,SKIP,SKIP,SKIP,0.069 s,0.02 MiB,0.04 MiB,37,36,SKIP,SKIP,SKIP,SKIP
|
||||
conv/multi_channel,arch-a,PASS,PASS,0.043 s,0.00 MiB,0.00 MiB,3,3,SKIP,SKIP,SKIP,0.046 s,0.00 MiB,0.00 MiB,4,3,SKIP,SKIP,SKIP,SKIP
|
||||
conv/non_square_kernel_1x3,arch-a,PASS,PASS,0.045 s,0.00 MiB,0.00 MiB,2,2,SKIP,SKIP,SKIP,0.047 s,0.00 MiB,0.00 MiB,3,2,SKIP,SKIP,SKIP,SKIP
|
||||
conv/non_square_kernel_3x1,arch-a,PASS,PASS,0.042 s,0.00 MiB,0.00 MiB,2,2,SKIP,SKIP,SKIP,0.046 s,0.00 MiB,0.00 MiB,3,2,SKIP,SKIP,SKIP,SKIP
|
||||
conv/non_uniform_stride,arch-a,PASS,PASS,0.045 s,0.00 MiB,0.00 MiB,3,3,SKIP,SKIP,SKIP,0.046 s,0.00 MiB,0.00 MiB,4,3,SKIP,SKIP,SKIP,SKIP
|
||||
conv/output_channel_grouping_minimal,arch-a,PASS,PASS,0.065 s,0.10 MiB,0.28 MiB,128,128,SKIP,SKIP,SKIP,0.104 s,0.10 MiB,0.25 MiB,128,128,SKIP,SKIP,SKIP,SKIP
|
||||
conv/pointwise_1x1,arch-a,PASS,PASS,0.044 s,0.00 MiB,0.00 MiB,1,1,SKIP,SKIP,SKIP,0.044 s,0.00 MiB,0.00 MiB,2,1,SKIP,SKIP,SKIP,SKIP
|
||||
conv/pointwise_tiled_chain,arch-a,PASS,PASS,0.738 s,0.01 MiB,0.03 MiB,17,80,SKIP,SKIP,SKIP,0.745 s,0.02 MiB,0.08 MiB,22,80,SKIP,SKIP,SKIP,SKIP
|
||||
conv/real_asymmetric_padding,arch-a,PASS,PASS,0.044 s,0.01 MiB,0.03 MiB,28,28,SKIP,SKIP,SKIP,0.060 s,0.01 MiB,0.03 MiB,29,28,SKIP,SKIP,SKIP,SKIP
|
||||
conv/relu_conv_store,arch-a,PASS,PASS,0.070 s,0.16 MiB,0.66 MiB,168,168,SKIP,SKIP,SKIP,0.120 s,0.17 MiB,0.66 MiB,166,166,SKIP,SKIP,SKIP,SKIP
|
||||
conv/same_lower_3x3,arch-a,PASS,PASS,0.043 s,0.01 MiB,0.02 MiB,25,25,SKIP,SKIP,SKIP,0.053 s,0.01 MiB,0.02 MiB,26,25,SKIP,SKIP,SKIP,SKIP
|
||||
conv/same_padding_3x3,arch-a,PASS,PASS,0.046 s,0.01 MiB,0.02 MiB,25,25,SKIP,SKIP,SKIP,0.056 s,0.01 MiB,0.02 MiB,26,25,SKIP,SKIP,SKIP,SKIP
|
||||
conv/strategy_depthwise_16,arch-a,PASS,PASS,0.062 s,0.06 MiB,0.33 MiB,168,168,SKIP,SKIP,SKIP,0.125 s,0.08 MiB,0.11 MiB,27,52,SKIP,SKIP,SKIP,SKIP
|
||||
conv/strategy_input_k_tiled,arch-a,PASS,PASS,0.061 s,0.08 MiB,0.27 MiB,108,108,SKIP,SKIP,SKIP,0.091 s,0.08 MiB,0.32 MiB,110,108,SKIP,SKIP,SKIP,SKIP
|
||||
conv/strategy_output_channel_tiled,arch-a,PASS,PASS,0.062 s,0.03 MiB,0.15 MiB,72,72,SKIP,SKIP,SKIP,0.078 s,0.04 MiB,0.13 MiB,73,72,SKIP,SKIP,SKIP,SKIP
|
||||
conv/strategy_streamed_packed,arch-a,PASS,PASS,0.131 s,3.34 MiB,7.89 MiB,168,168,SKIP,SKIP,SKIP,0.418 s,3.35 MiB,7.89 MiB,165,165,SKIP,SKIP,SKIP,SKIP
|
||||
conv/strategy_streamed_patch,arch-a,PASS,PASS,0.074 s,0.34 MiB,1.32 MiB,168,168,SKIP,SKIP,SKIP,0.159 s,0.35 MiB,1.32 MiB,165,165,SKIP,SKIP,SKIP,SKIP
|
||||
conv/strategy_tiled_2d,arch-a,PASS,PASS,0.096 s,0.10 MiB,0.37 MiB,168,168,SKIP,SKIP,SKIP,0.179 s,0.11 MiB,0.54 MiB,165,168,SKIP,SKIP,SKIP,SKIP
|
||||
conv/stride_2,arch-a,PASS,PASS,0.053 s,0.01 MiB,0.00 MiB,4,4,SKIP,SKIP,SKIP,0.053 s,0.01 MiB,0.00 MiB,6,4,SKIP,SKIP,SKIP,SKIP
|
||||
conv/with_bias_3x3,arch-a,PASS,PASS,0.045 s,0.00 MiB,0.01 MiB,3,3,SKIP,SKIP,SKIP,0.047 s,0.00 MiB,0.01 MiB,4,3,SKIP,SKIP,SKIP,SKIP
|
||||
conv/with_constant,arch-a,PASS,PASS,0.057 s,0.00 MiB,0.00 MiB,1,1,SKIP,SKIP,SKIP,0.069 s,0.00 MiB,0.00 MiB,3,1,SKIP,SKIP,SKIP,SKIP
|
||||
conv/without_kernel_shape_attr,arch-a,PASS,PASS,0.051 s,0.01 MiB,0.01 MiB,9,9,SKIP,SKIP,SKIP,0.057 s,0.01 MiB,0.01 MiB,11,9,SKIP,SKIP,SKIP,SKIP
|
||||
conv/yolo11n_depthwise_head,arch-a,PASS,PASS,2.129 s,8.66 MiB,33.48 MiB,164,250,SKIP,SKIP,SKIP,9.599 s,10.25 MiB,36.90 MiB,166,255,SKIP,SKIP,SKIP,SKIP
|
||||
conv/yolo11n_heavy,arch-a,PASS,PASS,0.188 s,4.82 MiB,19.00 MiB,160,800,SKIP,SKIP,SKIP,0.298 s,4.83 MiB,19.10 MiB,161,800,SKIP,SKIP,SKIP,SKIP
|
||||
conv/yolo11n_stem,arch-a,PASS,PASS,0.873 s,12.85 MiB,31.85 MiB,168,488,SKIP,SKIP,SKIP,1.318 s,12.88 MiB,31.68 MiB,165,485,SKIP,SKIP,SKIP,SKIP
|
||||
div/after_gemm,arch-a,PASS,PASS,0.046 s,0.01 MiB,0.01 MiB,4,4,SKIP,SKIP,SKIP,0.052 s,0.01 MiB,0.01 MiB,5,4,SKIP,SKIP,SKIP,SKIP
|
||||
div/basic,arch-a,PASS,PASS,0.042 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.044 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
div/channel_broadcast_1024,arch-a,PASS,PASS,0.048 s,0.02 MiB,0.01 MiB,1,0,SKIP,SKIP,SKIP,0.040 s,0.02 MiB,0.01 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
div/leading_dimension_broadcast,arch-a,PASS,PASS,0.053 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.058 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
div/runtime_scalar_rhs,arch-a,PASS,PASS,0.043 s,0.02 MiB,0.01 MiB,1,0,SKIP,SKIP,SKIP,0.050 s,0.02 MiB,0.01 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
div/scalar_constant,arch-a,PASS,PASS,0.047 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.043 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
gather/3d_input_axis1,arch-a,PASS,PASS,0.041 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.045 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
gather/axis0_matrix_indices,arch-a,PASS,PASS,0.040 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.039 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
gather/axis1,arch-a,PASS,PASS,0.040 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.040 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
gather/negative_axis,arch-a,PASS,PASS,0.041 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.042 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
gather/negative_indices,arch-a,PASS,PASS,0.061 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.049 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
gemm/alpha_beta,arch-a,PASS,PASS,0.050 s,0.01 MiB,0.01 MiB,4,4,SKIP,SKIP,SKIP,0.048 s,0.01 MiB,0.01 MiB,5,4,SKIP,SKIP,SKIP,SKIP
|
||||
gemm/bias_rank2_broadcast,arch-a,PASS,PASS,0.045 s,0.00 MiB,0.01 MiB,4,4,SKIP,SKIP,SKIP,0.062 s,0.01 MiB,0.01 MiB,5,4,SKIP,SKIP,SKIP,SKIP
|
||||
gemm/dynamic,arch-a,PASS,PASS,0.043 s,0.00 MiB,0.00 MiB,4,0,SKIP,SKIP,SKIP,0.049 s,0.00 MiB,0.00 MiB,5,0,SKIP,SKIP,SKIP,SKIP
|
||||
gemm/dynamic_alpha,arch-a,PASS,PASS,0.045 s,0.00 MiB,0.00 MiB,4,0,SKIP,SKIP,SKIP,0.045 s,0.00 MiB,0.00 MiB,5,0,SKIP,SKIP,SKIP,SKIP
|
||||
gemm/dynamic_beta,arch-a,PASS,PASS,0.057 s,0.00 MiB,0.00 MiB,4,0,SKIP,SKIP,SKIP,0.058 s,0.00 MiB,0.00 MiB,5,0,SKIP,SKIP,SKIP,SKIP
|
||||
gemm/dynamic_bias,arch-a,PASS,PASS,0.047 s,0.00 MiB,0.00 MiB,4,0,SKIP,SKIP,SKIP,0.053 s,0.00 MiB,0.00 MiB,5,0,SKIP,SKIP,SKIP,SKIP
|
||||
gemm/dynamic_bias_alpha_beta,arch-a,PASS,PASS,0.043 s,0.00 MiB,0.00 MiB,4,0,SKIP,SKIP,SKIP,0.046 s,0.00 MiB,0.00 MiB,5,0,SKIP,SKIP,SKIP,SKIP
|
||||
gemm/dynamic_transpose_b,arch-a,PASS,PASS,0.076 s,0.00 MiB,0.00 MiB,4,0,SKIP,SKIP,SKIP,0.077 s,0.00 MiB,0.00 MiB,5,0,SKIP,SKIP,SKIP,SKIP
|
||||
gemm/huge_1024,arch-a,PASS,PASS,0.186 s,0.01 MiB,0.09 MiB,64,64,SKIP,SKIP,SKIP,0.190 s,0.02 MiB,0.10 MiB,73,64,SKIP,SKIP,SKIP,SKIP
|
||||
gemm/large,arch-a,PASS,PASS,0.050 s,0.02 MiB,0.03 MiB,16,16,SKIP,SKIP,SKIP,0.048 s,0.02 MiB,0.03 MiB,17,16,SKIP,SKIP,SKIP,SKIP
|
||||
gemm/large_k_small_n,arch-a,PASS,PASS,0.113 s,0.01 MiB,0.01 MiB,8,8,SKIP,SKIP,SKIP,0.088 s,0.01 MiB,0.01 MiB,9,8,SKIP,SKIP,SKIP,SKIP
|
||||
gemm/non_square,arch-a,PASS,PASS,0.046 s,0.00 MiB,0.01 MiB,4,4,SKIP,SKIP,SKIP,0.049 s,0.01 MiB,0.01 MiB,5,4,SKIP,SKIP,SKIP,SKIP
|
||||
gemm/scalar_bias,arch-a,PASS,PASS,0.050 s,0.00 MiB,0.01 MiB,4,4,SKIP,SKIP,SKIP,0.057 s,0.01 MiB,0.01 MiB,5,4,SKIP,SKIP,SKIP,SKIP
|
||||
gemm/small,arch-a,PASS,PASS,0.047 s,0.00 MiB,0.00 MiB,2,2,SKIP,SKIP,SKIP,0.079 s,0.00 MiB,0.00 MiB,4,2,SKIP,SKIP,SKIP,SKIP
|
||||
gemm/small_k_large_n,arch-a,PASS,PASS,0.113 s,0.00 MiB,0.01 MiB,8,8,SKIP,SKIP,SKIP,0.110 s,0.01 MiB,0.02 MiB,9,8,SKIP,SKIP,SKIP,SKIP
|
||||
gemm/square_weights,arch-a,PASS,PASS,0.073 s,0.03 MiB,0.08 MiB,40,40,SKIP,SKIP,SKIP,0.084 s,0.03 MiB,0.09 MiB,43,40,SKIP,SKIP,SKIP,SKIP
|
||||
gemm/transpose_a,arch-a,PASS,PASS,0.055 s,0.00 MiB,0.01 MiB,4,4,SKIP,SKIP,SKIP,0.052 s,0.00 MiB,0.01 MiB,6,4,SKIP,SKIP,SKIP,SKIP
|
||||
gemm/transpose_a_and_b,arch-a,PASS,PASS,0.048 s,0.00 MiB,0.01 MiB,4,4,SKIP,SKIP,SKIP,0.053 s,0.00 MiB,0.01 MiB,6,4,SKIP,SKIP,SKIP,SKIP
|
||||
gemm/transpose_b,arch-a,PASS,PASS,0.065 s,0.00 MiB,0.01 MiB,4,4,SKIP,SKIP,SKIP,0.051 s,0.01 MiB,0.01 MiB,5,4,SKIP,SKIP,SKIP,SKIP
|
||||
gemm/transpose_b_with_bias,arch-a,PASS,PASS,0.053 s,0.01 MiB,0.01 MiB,4,4,SKIP,SKIP,SKIP,0.060 s,0.01 MiB,0.01 MiB,5,4,SKIP,SKIP,SKIP,SKIP
|
||||
gemm/with_bias,arch-a,PASS,PASS,0.054 s,0.01 MiB,0.01 MiB,4,4,SKIP,SKIP,SKIP,0.044 s,0.01 MiB,0.01 MiB,5,4,SKIP,SKIP,SKIP,SKIP
|
||||
gemv/all_constant,arch-a,PASS,PASS,0.047 s,0.00 MiB,0.00 MiB,0,0,SKIP,SKIP,SKIP,0.040 s,0.00 MiB,0.00 MiB,0,0,SKIP,SKIP,SKIP,SKIP
|
||||
gemv/constant_weight,arch-a,PASS,PASS,0.058 s,0.00 MiB,0.01 MiB,5,4,SKIP,SKIP,SKIP,0.059 s,0.00 MiB,0.01 MiB,7,4,SKIP,SKIP,SKIP,SKIP
|
||||
gemv/non_uniform_bias,arch-a,PASS,PASS,0.062 s,0.00 MiB,0.01 MiB,5,4,SKIP,SKIP,SKIP,0.061 s,0.00 MiB,0.01 MiB,7,4,SKIP,SKIP,SKIP,SKIP
|
||||
gemv/scalar_bias,arch-a,PASS,PASS,0.061 s,0.00 MiB,0.01 MiB,5,4,SKIP,SKIP,SKIP,0.072 s,0.00 MiB,0.01 MiB,7,4,SKIP,SKIP,SKIP,SKIP
|
||||
gemv/uniform_bias,arch-a,PASS,PASS,0.076 s,0.00 MiB,0.01 MiB,5,4,SKIP,SKIP,SKIP,0.064 s,0.00 MiB,0.01 MiB,7,4,SKIP,SKIP,SKIP,SKIP
|
||||
matmul/basic,arch-a,PASS,PASS,0.071 s,0.00 MiB,0.00 MiB,2,2,SKIP,SKIP,SKIP,0.072 s,0.00 MiB,0.00 MiB,4,2,SKIP,SKIP,SKIP,SKIP
|
||||
matmul/batched_3d,arch-a,PASS,PASS,0.066 s,0.00 MiB,0.01 MiB,4,4,SKIP,SKIP,SKIP,0.060 s,0.00 MiB,0.01 MiB,5,4,SKIP,SKIP,SKIP,SKIP
|
||||
matmul/batched_3d_dynamic,arch-a,PASS,PASS,0.048 s,0.00 MiB,0.00 MiB,4,0,SKIP,SKIP,SKIP,0.042 s,0.00 MiB,0.00 MiB,4,0,SKIP,SKIP,SKIP,SKIP
|
||||
matmul/batched_left_constant,arch-a,PASS,PASS,0.048 s,0.00 MiB,0.02 MiB,8,8,SKIP,SKIP,SKIP,0.054 s,0.01 MiB,0.02 MiB,9,8,SKIP,SKIP,SKIP,SKIP
|
||||
matmul/batched_lhs_broadcast,arch-a,PASS,PASS,0.047 s,0.00 MiB,0.01 MiB,4,4,SKIP,SKIP,SKIP,0.056 s,0.00 MiB,0.01 MiB,5,4,SKIP,SKIP,SKIP,SKIP
|
||||
matmul/batched_rhs_broadcast,arch-a,PASS,PASS,0.052 s,0.00 MiB,0.01 MiB,4,4,SKIP,SKIP,SKIP,0.053 s,0.00 MiB,0.01 MiB,5,4,SKIP,SKIP,SKIP,SKIP
|
||||
matmul/dynamic,arch-a,PASS,PASS,0.044 s,0.00 MiB,0.00 MiB,4,0,SKIP,SKIP,SKIP,0.054 s,0.00 MiB,0.00 MiB,5,0,SKIP,SKIP,SKIP,SKIP
|
||||
matmul/huge_1024,arch-a,PASS,PASS,0.174 s,0.01 MiB,0.09 MiB,64,64,SKIP,SKIP,SKIP,0.188 s,0.02 MiB,0.10 MiB,73,64,SKIP,SKIP,SKIP,SKIP
|
||||
matmul/left_constant,arch-a,PASS,PASS,0.051 s,0.00 MiB,0.01 MiB,4,4,SKIP,SKIP,SKIP,0.058 s,0.00 MiB,0.01 MiB,6,4,SKIP,SKIP,SKIP,SKIP
|
||||
matmul/matrix_vector,arch-a,PASS,PASS,0.120 s,0.52 MiB,0.77 MiB,165,170,SKIP,SKIP,SKIP,0.239 s,0.53 MiB,0.77 MiB,165,173,SKIP,SKIP,SKIP,SKIP
|
||||
matmul/vector_matrix,arch-a,PASS,PASS,0.088 s,0.01 MiB,0.01 MiB,8,8,SKIP,SKIP,SKIP,0.087 s,0.01 MiB,0.01 MiB,9,8,SKIP,SKIP,SKIP,SKIP
|
||||
matmul/yolo_attention,arch-a,PASS,PASS,0.394 s,1.02 MiB,39.22 MiB,168,0,SKIP,SKIP,SKIP,0.512 s,2.65 MiB,26.70 MiB,43,0,SKIP,SKIP,SKIP,SKIP
|
||||
mul/after_conv,arch-a,PASS,PASS,0.065 s,0.00 MiB,0.00 MiB,3,3,SKIP,SKIP,SKIP,0.081 s,0.00 MiB,0.00 MiB,4,3,SKIP,SKIP,SKIP,SKIP
|
||||
mul/after_conv_scalar_constant,arch-a,PASS,PASS,0.053 s,0.00 MiB,0.00 MiB,3,3,SKIP,SKIP,SKIP,0.055 s,0.00 MiB,0.00 MiB,4,3,SKIP,SKIP,SKIP,SKIP
|
||||
mul/basic,arch-a,PASS,PASS,0.075 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.079 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
mul/channel_broadcast_1024,arch-a,PASS,PASS,0.038 s,0.02 MiB,0.01 MiB,1,0,SKIP,SKIP,SKIP,0.039 s,0.02 MiB,0.01 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
mul/leading_dimension_broadcast,arch-a,PASS,PASS,0.038 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.037 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
mul/scalar_constant,arch-a,PASS,PASS,0.038 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.036 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
pool/avg_basic,arch-a,PASS,PASS,0.047 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.045 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
pool/avg_ceil_mode,arch-a,PASS,PASS,0.061 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.056 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
pool/avg_explicit_padding,arch-a,PASS,PASS,0.047 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.051 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
pool/avg_include_pad,arch-a,PASS,PASS,0.049 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.050 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
pool/avg_large_channels,arch-a,PASS,PASS,0.056 s,0.04 MiB,0.02 MiB,1,0,SKIP,SKIP,SKIP,0.059 s,0.04 MiB,0.02 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
pool/avg_non_uniform_stride,arch-a,PASS,PASS,0.050 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.053 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
pool/avg_real_asymmetric_padding,arch-a,PASS,PASS,0.048 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.048 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
pool/max_after_conv,arch-a,PASS,PASS,0.052 s,0.00 MiB,0.00 MiB,4,4,SKIP,SKIP,SKIP,0.054 s,0.00 MiB,0.00 MiB,5,4,SKIP,SKIP,SKIP,SKIP
|
||||
pool/max_basic,arch-a,PASS,PASS,0.040 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.039 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
pool/max_ceil_mode,arch-a,PASS,PASS,0.042 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.044 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
pool/max_global_style_kernel_equals_input,arch-a,PASS,PASS,0.050 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.041 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
pool/max_non_square_kernel,arch-a,PASS,PASS,0.044 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.067 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
pool/max_real_asymmetric_padding,arch-a,PASS,PASS,0.059 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.059 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
pool/max_same_upper,arch-a,PASS,PASS,0.059 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.049 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
pool/max_stride2_multichannel,arch-a,PASS,PASS,0.045 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.062 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
reduce_mean/4d_spatial,arch-a,PASS,PASS,0.044 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.046 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
reduce_mean/4d_spatial_keepdims_0,arch-a,PASS,PASS,0.044 s,0.00 MiB,0.00 MiB,3,0,SKIP,SKIP,SKIP,0.045 s,0.00 MiB,0.00 MiB,4,0,SKIP,SKIP,SKIP,SKIP
|
||||
reduce_mean/after_conv,arch-a,PASS,PASS,0.069 s,0.00 MiB,0.00 MiB,3,3,SKIP,SKIP,SKIP,0.067 s,0.00 MiB,0.00 MiB,4,3,SKIP,SKIP,SKIP,SKIP
|
||||
reduce_mean/all_axes_keepdims_0,arch-a,PASS,PASS,0.044 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.048 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
reduce_mean/all_axes_keepdims_1,arch-a,PASS,PASS,0.041 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.052 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
reduce_mean/basic,arch-a,PASS,PASS,0.048 s,0.00 MiB,0.00 MiB,4,0,SKIP,SKIP,SKIP,0.042 s,0.00 MiB,0.00 MiB,4,0,SKIP,SKIP,SKIP,SKIP
|
||||
reduce_mean/channel_axis_nchw,arch-a,PASS,PASS,0.053 s,0.03 MiB,0.02 MiB,4,0,SKIP,SKIP,SKIP,0.048 s,0.03 MiB,0.02 MiB,4,0,SKIP,SKIP,SKIP,SKIP
|
||||
reduce_mean/keepdims_0,arch-a,PASS,PASS,0.046 s,0.00 MiB,0.00 MiB,4,0,SKIP,SKIP,SKIP,0.041 s,0.00 MiB,0.00 MiB,5,0,SKIP,SKIP,SKIP,SKIP
|
||||
reduce_mean/large_dimension_1024,arch-a,PASS,PASS,0.037 s,0.01 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.039 s,0.01 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
reduce_mean/legacy_axes_1_2_keepdims_1,arch-a,PASS,PASS,0.041 s,0.00 MiB,0.00 MiB,2,0,SKIP,SKIP,SKIP,0.051 s,0.00 MiB,0.00 MiB,2,0,SKIP,SKIP,SKIP,SKIP
|
||||
reduce_mean/legacy_axis1_keepdims_0,arch-a,PASS,PASS,0.049 s,0.00 MiB,0.00 MiB,8,0,SKIP,SKIP,SKIP,0.043 s,0.00 MiB,0.00 MiB,9,0,SKIP,SKIP,SKIP,SKIP
|
||||
reduce_mean/legacy_axis1_keepdims_1,arch-a,PASS,PASS,0.043 s,0.00 MiB,0.00 MiB,8,0,SKIP,SKIP,SKIP,0.041 s,0.00 MiB,0.00 MiB,8,0,SKIP,SKIP,SKIP,SKIP
|
||||
reduce_mean/legacy_empty_axes_noop,arch-a,PASS,PASS,0.035 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.040 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
reduce_mean/legacy_nchw_spatial,arch-a,PASS,PASS,0.048 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.055 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
reduce_mean/legacy_negative_axis,arch-a,PASS,PASS,0.050 s,0.00 MiB,0.00 MiB,6,0,SKIP,SKIP,SKIP,0.064 s,0.00 MiB,0.00 MiB,6,0,SKIP,SKIP,SKIP,SKIP
|
||||
reduce_mean/legacy_reduce_all_keepdims_1,arch-a,PASS,PASS,0.040 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.041 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
reduce_mean/negative_axis,arch-a,PASS,PASS,0.044 s,0.00 MiB,0.00 MiB,6,0,SKIP,SKIP,SKIP,0.046 s,0.00 MiB,0.00 MiB,6,0,SKIP,SKIP,SKIP,SKIP
|
||||
relu/4d,arch-a,PASS,PASS,0.043 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.039 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
relu/after_conv,arch-a,PASS,PASS,0.061 s,0.00 MiB,0.00 MiB,3,3,SKIP,SKIP,SKIP,0.062 s,0.00 MiB,0.00 MiB,4,3,SKIP,SKIP,SKIP,SKIP
|
||||
relu/after_gemm,arch-a,PASS,PASS,0.044 s,0.01 MiB,0.01 MiB,4,4,SKIP,SKIP,SKIP,0.045 s,0.01 MiB,0.01 MiB,5,4,SKIP,SKIP,SKIP,SKIP
|
||||
relu/basic,arch-a,PASS,PASS,0.039 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.038 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
reshape/4d_to_2d_flatten,arch-a,PASS,PASS,0.042 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.045 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
reshape/infer_dim_minus_one,arch-a,PASS,PASS,0.041 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.040 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
reshape/same_rank,arch-a,PASS,PASS,0.040 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.039 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
reshape/zero_copies_input_dim,arch-a,PASS,PASS,0.042 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.036 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
resize/height_only,arch-a,PASS,PASS,0.047 s,0.00 MiB,0.00 MiB,4,0,SKIP,SKIP,SKIP,0.047 s,0.00 MiB,0.00 MiB,4,0,SKIP,SKIP,SKIP,SKIP
|
||||
resize/nearest_2x,arch-a,PASS,PASS,0.042 s,0.00 MiB,0.00 MiB,4,0,SKIP,SKIP,SKIP,0.051 s,0.00 MiB,0.00 MiB,4,0,SKIP,SKIP,SKIP,SKIP
|
||||
resize/nearest_downsample,arch-a,PASS,PASS,0.042 s,0.00 MiB,0.00 MiB,2,0,SKIP,SKIP,SKIP,0.045 s,0.00 MiB,0.00 MiB,2,0,SKIP,SKIP,SKIP,SKIP
|
||||
resize/non_uniform_scales,arch-a,PASS,PASS,0.042 s,0.00 MiB,0.00 MiB,6,0,SKIP,SKIP,SKIP,0.046 s,0.00 MiB,0.00 MiB,6,0,SKIP,SKIP,SKIP,SKIP
|
||||
resize/width_only,arch-a,PASS,PASS,0.043 s,0.00 MiB,0.00 MiB,2,0,SKIP,SKIP,SKIP,0.043 s,0.00 MiB,0.00 MiB,2,0,SKIP,SKIP,SKIP,SKIP
|
||||
resize/with_sizes,arch-a,PASS,PASS,0.045 s,0.00 MiB,0.00 MiB,3,0,SKIP,SKIP,SKIP,0.047 s,0.00 MiB,0.00 MiB,3,0,SKIP,SKIP,SKIP,SKIP
|
||||
sigmoid/4d,arch-a,PASS,PASS,0.040 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.058 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
sigmoid/after_gemm,arch-a,PASS,PASS,0.048 s,0.01 MiB,0.01 MiB,4,4,SKIP,SKIP,SKIP,0.051 s,0.01 MiB,0.01 MiB,5,4,SKIP,SKIP,SKIP,SKIP
|
||||
sigmoid/basic,arch-a,PASS,PASS,0.038 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.053 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
slice/2d_basic,arch-a,PASS,PASS,0.039 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.043 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
slice/after_conv,arch-a,PASS,PASS,0.050 s,0.00 MiB,0.01 MiB,6,6,SKIP,SKIP,SKIP,0.059 s,0.00 MiB,0.01 MiB,7,6,SKIP,SKIP,SKIP,SKIP
|
||||
slice/default_axes,arch-a,PASS,PASS,0.039 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.038 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
slice/large_channel_1024,arch-a,PASS,PASS,0.041 s,0.01 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.039 s,0.01 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
slice/nchw_spatial_crop,arch-a,PASS,PASS,0.044 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.044 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
slice/negative_axis,arch-a,PASS,PASS,0.055 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.042 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
slice/negative_indices,arch-a,PASS,PASS,0.043 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.038 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
slice/step2,arch-a,PASS,PASS,0.041 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.042 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
softmax/3d_last_axis,arch-a,PASS,PASS,0.044 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.042 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
softmax/basic,arch-a,PASS,PASS,0.040 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.038 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
softmax/channel_axis,arch-a,PASS,PASS,0.037 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.047 s,0.00 MiB,0.00 MiB,3,0,SKIP,SKIP,SKIP,SKIP
|
||||
softmax/large_dimension_1024,arch-a,PASS,PASS,0.036 s,0.01 MiB,0.01 MiB,1,0,SKIP,SKIP,SKIP,0.038 s,0.01 MiB,0.01 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
softmax/negative_axis,arch-a,PASS,PASS,0.069 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.060 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
split/basic,arch-a,PASS,PASS,0.052 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.057 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
split/equal_three_way,arch-a,PASS,PASS,0.042 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.042 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
split/negative_axis,arch-a,PASS,PASS,0.058 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.043 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
split/uneven_channel_axis_4d,arch-a,PASS,PASS,0.048 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.040 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
sub/after_gemm,arch-a,PASS,PASS,0.050 s,0.01 MiB,0.01 MiB,4,4,SKIP,SKIP,SKIP,0.047 s,0.01 MiB,0.01 MiB,5,4,SKIP,SKIP,SKIP,SKIP
|
||||
sub/basic,arch-a,PASS,PASS,0.040 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.043 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
sub/broadcast_row,arch-a,PASS,PASS,0.056 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.044 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
sub/channel_broadcast_1024,arch-a,PASS,PASS,0.052 s,0.02 MiB,0.01 MiB,1,0,SKIP,SKIP,SKIP,0.043 s,0.02 MiB,0.01 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
sub/constant_lhs_broadcast,arch-a,PASS,PASS,0.044 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.038 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
sub/leading_dimension_broadcast,arch-a,PASS,PASS,0.040 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.033 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP
|
||||
|
@@ -3,7 +3,6 @@ from __future__ import annotations
|
||||
import argparse
|
||||
import copy
|
||||
import csv
|
||||
import importlib.util
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
@@ -48,26 +47,6 @@ def test_ablation_results_stay_in_independent_output_roots(tmp_path):
|
||||
assert rows[1]["latency"] == change
|
||||
|
||||
|
||||
@pytest.mark.parametrize("custom_input", [False, True])
|
||||
def test_ablation_plots_follow_the_selected_experiment(tmp_path, monkeypatch, custom_input):
|
||||
spec = importlib.util.spec_from_file_location("ablation_plots", TOOLS / "ablation/generate_plots.py")
|
||||
plots = importlib.util.module_from_spec(spec)
|
||||
monkeypatch.setitem(sys.modules, spec.name, plots)
|
||||
spec.loader.exec_module(plots)
|
||||
expected = tmp_path / "experiment/results_ablation.csv" if custom_input else (
|
||||
ablation.DEFAULT_RESULTS_PATH
|
||||
)
|
||||
monkeypatch.setattr(sys, "argv", ["generate_plots.py", *(["--input", str(expected)] if custom_input else [])])
|
||||
result = plots.Result("baseline", "arch-a", "vgg8", "latency", 1, "PASS", {})
|
||||
inputs, outputs = [], []
|
||||
monkeypatch.setattr(plots, "configure_style", lambda: None)
|
||||
monkeypatch.setattr(plots, "load_results", lambda path: inputs.append(path) or [result])
|
||||
monkeypatch.setattr(plots, "draw_metric", lambda rows, metric, label, output: outputs.append(output))
|
||||
plots.main()
|
||||
assert inputs == [expected]
|
||||
assert outputs == [expected.parent / "plots"] * 3
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def recorded_failure(tmp_path):
|
||||
model, functional, config = (tmp_path / name for name in ("model.onnx", "functional.onnx", "config.json"))
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import csv
|
||||
from dataclasses import replace
|
||||
import importlib.util
|
||||
import math
|
||||
from pathlib import Path
|
||||
@@ -14,16 +15,14 @@ TOOLS = Path(__file__).resolve().parents[1] / "tools/pim"
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def plotters(monkeypatch):
|
||||
modules = []
|
||||
for name, directory in (("comparison_plots", "pimcomp/compare"), ("ablation_plots", "ablation")):
|
||||
monkeypatch.syspath_prepend(str(TOOLS / directory))
|
||||
spec = importlib.util.spec_from_file_location(name, TOOLS / directory / "generate_plots.py")
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
monkeypatch.setitem(sys.modules, name, module)
|
||||
spec.loader.exec_module(module)
|
||||
modules.append(module)
|
||||
return modules
|
||||
def comparison(monkeypatch):
|
||||
directory = TOOLS / "pimcomp/compare"
|
||||
monkeypatch.syspath_prepend(str(directory))
|
||||
spec = importlib.util.spec_from_file_location("comparison_plots", directory / "generate_plots.py")
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
monkeypatch.setitem(sys.modules, spec.name, module)
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
@@ -42,8 +41,7 @@ def write_csv(path, fields, rows):
|
||||
|
||||
|
||||
@pytest.mark.parametrize("custom", [False, True])
|
||||
def test_comparison_plot_cli_input_paths(plotters, tmp_path, monkeypatch, custom):
|
||||
comparison, _ = plotters
|
||||
def test_comparison_plot_cli_input_paths(comparison, tmp_path, monkeypatch, custom):
|
||||
expected = tmp_path / "custom.csv" if custom else TOOLS / "pimcomp/compare/results_comparison.csv"
|
||||
seen = []
|
||||
|
||||
@@ -62,8 +60,7 @@ def test_comparison_plot_cli_input_paths(plotters, tmp_path, monkeypatch, custom
|
||||
assert seen == [expected]
|
||||
|
||||
|
||||
def test_exact_comparison_configuration_matrix(plotters, comparison_csv, tmp_path):
|
||||
comparison, _ = plotters
|
||||
def test_exact_comparison_configuration_matrix(comparison, comparison_csv, tmp_path):
|
||||
fields, rows = comparison_csv
|
||||
results = comparison.load_results(write_csv(tmp_path / "results.csv", fields, rows))
|
||||
keys = {(row.arch, row.model, row.mode, row.pipeline) for row in results}
|
||||
@@ -74,8 +71,7 @@ def test_exact_comparison_configuration_matrix(plotters, comparison_csv, tmp_pat
|
||||
@pytest.mark.parametrize("damage,message", [("missing", "Missing"), ("duplicate", "Duplicate"),
|
||||
("arch", "Unexpected"), ("model", "Unexpected"),
|
||||
("pipeline", "Unexpected"), ("mode", "Unexpected")])
|
||||
def test_comparison_rejects_bad_matrix_keys(plotters, comparison_csv, tmp_path, damage, message):
|
||||
comparison, _ = plotters
|
||||
def test_comparison_rejects_bad_matrix_keys(comparison, comparison_csv, tmp_path, damage, message):
|
||||
fields, rows = comparison_csv
|
||||
if damage == "missing":
|
||||
rows.pop()
|
||||
@@ -108,50 +104,431 @@ else:
|
||||
@pytest.mark.parametrize("field,value", [("pimcomp_latency_ms", "999"),
|
||||
("pimcomp_throughput_samples_s", "999"), ("pimcomp_power_mw", "999"),
|
||||
("pimcomp_energy_pj", "999"), ("pimcomp_functional_validation", "FAIL")])
|
||||
def test_repeated_pimcomp_batch_results_must_agree(plotters, comparison_csv, tmp_path, field, value):
|
||||
comparison, _ = plotters
|
||||
def test_repeated_pimcomp_batch_results_must_agree(comparison, comparison_csv, tmp_path, field, value):
|
||||
fields, rows = comparison_csv
|
||||
row = next(row for row in rows if row["arch"] == "arch-a" and row["model"] == "vgg8"
|
||||
and row["raptor_pipeline"] == "4")
|
||||
row[field] = value
|
||||
with pytest.raises(ValueError, match="inconsistent shared Pimcomp batch results"):
|
||||
with pytest.raises(ValueError, match="inconsistent shared PIMCOMP batch results"):
|
||||
comparison.load_results(write_csv(tmp_path / "results.csv", fields, rows))
|
||||
|
||||
|
||||
def test_legacy_invalid_ablation_pairs_are_unavailable_without_changing_pass(plotters):
|
||||
_, ablation = plotters
|
||||
with pytest.warns(UserWarning, match="2 invalid reciprocal"):
|
||||
rows = ablation.load_results(TOOLS / "ablation/results_ablation.csv")
|
||||
invalid = [row for row in rows if row.model == "googlenet" and row.variant == "no-plan"
|
||||
and row.mode == "throughput"]
|
||||
assert len(invalid) == 2
|
||||
assert all(row.validation == "PASS" and math.isnan(row.metrics["latency"])
|
||||
and math.isnan(row.metrics["throughput"]) for row in invalid)
|
||||
@pytest.mark.parametrize("status,expected", [("PASS", ["1.0"]),
|
||||
("FAIL", ["1.0", "INVALID"]), ("SKIPPED", ["1.0", "SKIPPED"]),
|
||||
("NA", ["1.0", "NA"])])
|
||||
@pytest.mark.parametrize("direction", [1, -1])
|
||||
def test_value_label_validation_text(comparison, status, expected, direction):
|
||||
fig, ax = comparison.plt.subplots()
|
||||
try:
|
||||
comparison.value_label(ax, 0, 1, "1.0", status, direction)
|
||||
assert [text.get_text() for text in ax.texts] == expected
|
||||
assert ax.texts[-1].get_color() == (comparison.ERROR_COLOR if status == "FAIL" else comparison.TEXT_COLOR)
|
||||
if status != "PASS":
|
||||
value, validation = ax.texts
|
||||
assert value.xy == validation.xy
|
||||
assert value.get_position()[1] == validation.get_position()[1] == 0
|
||||
assert value.get_position()[0] == -validation.get_position()[0] != 0
|
||||
finally:
|
||||
comparison.plt.close(fig)
|
||||
|
||||
|
||||
def test_ablation_missing_values_and_legitimate_extremes(plotters, tmp_path):
|
||||
_, ablation = plotters
|
||||
row = {"variant": "no-plan", "arch": "arch-a", "model": "vgg8", "mode": "throughput",
|
||||
"raptor_pipeline": "4", "functional_validation": "FAIL", "latency": "-100.00%",
|
||||
"throughput": "+100000000.00%", "power": "NA", "energy": "-100.00%"}
|
||||
result = ablation.load_results(write_csv(tmp_path / "extreme.csv", list(row), [row]))[0]
|
||||
assert result.validation == "FAIL"
|
||||
assert result.metrics["latency"] == result.metrics["energy"] == -100
|
||||
assert result.metrics["throughput"] == 100000000
|
||||
assert math.isnan(result.metrics["power"])
|
||||
@pytest.mark.parametrize("values", [(1, 2), (-1, -2), (-1, 2), (0, 0)])
|
||||
def test_value_label_limits_are_tight_shared_and_unclipped(comparison, values):
|
||||
fig, axes = comparison.plt.subplots(1, 2, sharey=True, figsize=(10, 4.4), layout="constrained")
|
||||
try:
|
||||
for ax, value in zip(axes, values):
|
||||
ax.set_ylim(-10, 10)
|
||||
ax.bar(0, value)
|
||||
direction = -1 if value < 0 else 1
|
||||
comparison.value_label(ax, 0, value + direction * 0.05, f"{value:+.2f}%", "FAIL", direction)
|
||||
comparison.fit_value_labels(fig)
|
||||
fig.canvas.draw()
|
||||
renderer = fig.canvas.get_renderer()
|
||||
assert axes[0].get_ylim() == axes[1].get_ylim()
|
||||
lower, upper = axes[0].get_ylim()
|
||||
assert upper - lower < 20
|
||||
assert lower <= min(0, *values) and upper >= max(0, *values)
|
||||
assert axes[0].get_window_extent(renderer).height == pytest.approx(axes[1].get_window_extent(renderer).height)
|
||||
for ax in axes:
|
||||
bounds = ax.get_window_extent(renderer)
|
||||
for text in ax.texts:
|
||||
extent = text.get_window_extent(renderer)
|
||||
assert bounds.y0 <= extent.y0 <= extent.y1 <= bounds.y1
|
||||
finally:
|
||||
comparison.plt.close(fig)
|
||||
|
||||
|
||||
def test_both_plotters_export_png_and_svg(plotters, tmp_path):
|
||||
comparison, ablation = plotters
|
||||
@pytest.mark.parametrize("statuses,expected", [
|
||||
(("PASS", "PASS", "PASS"), "PASS"),
|
||||
(("FAIL", "PASS", "PASS"), "FAIL"),
|
||||
(("PASS", "FAIL", "PASS"), "FAIL"),
|
||||
(("PASS", "PASS", "FAIL"), "FAIL"),
|
||||
(("PASS", "SKIPPED", "NA"), "SKIPPED"),
|
||||
(("PASS", "PASS", "NA"), "N/A"),
|
||||
])
|
||||
def test_compact_validation_combines_pipeline_statuses(comparison, tmp_path, monkeypatch, statuses, expected):
|
||||
rows = comparison.load_results(TOOLS / "pimcomp/compare/results_comparison.csv")
|
||||
stages = dict(zip((2, 4, 8), statuses))
|
||||
rows = [replace(row, raptor_validation=stages[row.pipeline])
|
||||
if row.arch == "arch-a" and row.model == "vgg8" and row.mode == "throughput"
|
||||
else row for row in rows]
|
||||
captured = []
|
||||
monkeypatch.setattr(comparison, "save_figure", lambda fig, name, path: captured.append((fig, name)))
|
||||
comparison.draw_validation(rows, tmp_path, compact=True)
|
||||
fig, name = captured[0]
|
||||
try:
|
||||
assert name == "validation_status_compact"
|
||||
assert fig.get_size_inches() == pytest.approx((8.5, 3.3))
|
||||
for index, ax in enumerate(fig.axes):
|
||||
assert [label.get_text() for label in ax.get_xticklabels(minor=True)] == [
|
||||
"Raptor", "PIMCOMP", "Raptor", "PIMCOMP"]
|
||||
assert [label.get_text() for label in ax.get_yticklabels()] == ([
|
||||
"VGG-8", "ResNet-18", "ResNet-34", "GoogLeNet", "YOLO11n"] if index == 0 else [])
|
||||
assert len(ax.patches) == len(ax.texts) == 4 * len(comparison.MODELS)
|
||||
assert [label.get_text() for label in ax.get_xticklabels()] == ["Latency", "Throughput"]
|
||||
assert all(text.get_fontsize() == 12 and text.get_weight() == "normal" for text in ax.texts)
|
||||
assert all(text.get_color() == "#1D2530" for text in ax.texts)
|
||||
ax = fig.axes[0]
|
||||
throughput_x = ax.xaxis.get_minorticklocs()[2]
|
||||
assert [text.get_text() for text in ax.texts if text.get_position() == (throughput_x, 0)] == [expected]
|
||||
finally:
|
||||
comparison.plt.close(fig)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("rates,stage", [((300, 200, 100), 2), ((100, 300, 200), 4),
|
||||
((100, 200, 300), 8), ((math.nan, 200, 300), 8), ((300, 300, 200), 2),
|
||||
((math.nan, math.nan, math.nan), 2)])
|
||||
def test_compact_speedups_select_best_pipeline(comparison, rates, stage):
|
||||
template = comparison.load_results(TOOLS / "pimcomp/compare/results_comparison.csv")[0]
|
||||
rows = [replace(template, latency_speedup=0.5)]
|
||||
rows += [replace(template, mode="throughput", pipeline=depth, raptor_throughput=rate,
|
||||
pimcomp_throughput=100, raptor_validation="FAIL")
|
||||
for depth, rate in zip((2, 4, 8), rates)]
|
||||
latency, throughput = comparison.select_compact_speedups(rows)
|
||||
assert latency is rows[0] and latency.speedup == 0.5
|
||||
assert throughput.pipeline == stage and throughput.raptor_validation == "FAIL"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("diverging", [False, True])
|
||||
@pytest.mark.parametrize("metric,ylabel", [("speedup", "Speedup (x)"),
|
||||
("energy", "Energy-per-inference advantage (x)"), ("power", "Average power advantage (x)")])
|
||||
def test_compact_metric_bars_and_units(comparison, tmp_path, monkeypatch, diverging, metric, ylabel):
|
||||
from matplotlib.colors import to_rgba
|
||||
|
||||
template = comparison.load_results(TOOLS / "pimcomp/compare/results_comparison.csv")[0]
|
||||
rows = [replace(template, arch=arch, mode=mode, pipeline=stage, latency_speedup=0.5,
|
||||
energy=0.5 if mode == "latency" else 3, power=0.5 if mode == "latency" else 3,
|
||||
raptor_throughput=300, pimcomp_throughput=100, pimcomp_validation="FAIL")
|
||||
for arch in comparison.ARCHITECTURES for mode, stage in (("latency", 1), ("throughput", 4))]
|
||||
captured = []
|
||||
monkeypatch.setattr(comparison, "save_figure", lambda fig, name, path: captured.append(fig))
|
||||
comparison.draw_metric(rows, metric, ylabel, "compact", tmp_path,
|
||||
diverging=diverging, compact=True)
|
||||
fig = captured[0]
|
||||
try:
|
||||
assert fig.get_size_inches() == pytest.approx((8.5, 4.4))
|
||||
expected_legend = {
|
||||
"speedup": ["Single-inference latency", "Best-case throughput"],
|
||||
"energy": ["Single-inference energy", "Best-case throughput energy"],
|
||||
"power": ["Single-inference power", "Best-case throughput power"],
|
||||
}
|
||||
assert [text.get_text() for text in fig.legends[0].get_texts()] == expected_legend[metric]
|
||||
for index, ax in enumerate(fig.axes):
|
||||
latency, throughput = (container.patches[0] for container in ax.containers)
|
||||
lower, upper = ax.get_xlim()
|
||||
assert latency.get_x() - lower == pytest.approx(0.4)
|
||||
assert upper - throughput.get_x() - throughput.get_width() == pytest.approx(0.4)
|
||||
assert latency.get_x() + latency.get_width() == throughput.get_x()
|
||||
assert latency.get_facecolor() == to_rgba(comparison.METRIC_STYLES["latency"][0])
|
||||
assert throughput.get_facecolor() == to_rgba(comparison.METRIC_STYLES["throughput"][0])
|
||||
assert latency.get_edgecolor() == to_rgba(comparison.METRIC_STYLES["latency"][1])
|
||||
assert throughput.get_edgecolor() == to_rgba(comparison.METRIC_STYLES["throughput"][1])
|
||||
assert not latency.get_hatch() and throughput.get_hatch() == "//"
|
||||
assert latency.get_height() == (-1 if diverging else 0.5)
|
||||
assert throughput.get_height() == (2 if diverging else 3)
|
||||
assert ax.get_ylabel() == (ylabel if index == 0 else "")
|
||||
assert bool(ax.get_yticklabels()) == (index == 0)
|
||||
assert all("×" not in label.get_text() for label in ax.get_yticklabels())
|
||||
assert [text.get_text() for text in ax.texts] == [
|
||||
"2.00×" if diverging else "0.50×", "P. INVALID", "3.00×", "P. INVALID"]
|
||||
finally:
|
||||
comparison.plt.close(fig)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("missing_energy", [False, True])
|
||||
def test_compact_metrics_use_speedup_pipeline_selection(comparison, monkeypatch, missing_energy):
|
||||
rows = comparison.load_results(TOOLS / "pimcomp/compare/results_comparison.csv")
|
||||
cases = {2: (100, 9, 8), 4: (300, math.nan if missing_energy else 0.5, 0.6), 8: (200, 7, 6)}
|
||||
rows = [replace(row, raptor_throughput=cases[row.pipeline][0],
|
||||
energy=cases[row.pipeline][1], power=cases[row.pipeline][2])
|
||||
if row.arch == "arch-a" and row.model == "vgg8" and row.mode == "throughput"
|
||||
else row for row in rows]
|
||||
captured = []
|
||||
|
||||
def capture(selected, field, ylabel, basename, output_dir, diverging=False, compact=False):
|
||||
if compact:
|
||||
captured.append((selected, field, basename, diverging))
|
||||
|
||||
monkeypatch.setattr(comparison, "load_results", lambda path: rows)
|
||||
for name in ("configure_style", "draw_validation", "draw_pipeline_effect", "draw_absolute"):
|
||||
monkeypatch.setattr(comparison, name, lambda *args, **kwargs: None)
|
||||
monkeypatch.setattr(comparison, "draw_metric", capture)
|
||||
monkeypatch.setattr(sys, "argv", ["generate_plots.py"])
|
||||
comparison.main()
|
||||
assert {(field, basename, diverging) for _, field, basename, diverging in captured} == {
|
||||
(metric, f"{metric}_compact_{suffix}", diverging)
|
||||
for metric in ("speedup", "energy", "power")
|
||||
for suffix, diverging in (("comparison", False), ("diverging", True))}
|
||||
assert len(captured) == 6
|
||||
selected = captured[0][0]
|
||||
assert all(plotted is selected for plotted, _, _, _ in captured)
|
||||
throughput = next(row for row in selected if row.arch == "arch-a" and row.model == "vgg8"
|
||||
and row.mode == "throughput")
|
||||
assert throughput.pipeline == 4 and throughput.power == 0.6
|
||||
assert math.isnan(throughput.energy) if missing_energy else throughput.energy == 0.5
|
||||
|
||||
|
||||
def test_compiler_palette_and_grayscale_patterns(comparison, tmp_path, monkeypatch):
|
||||
from matplotlib.colors import to_rgba
|
||||
|
||||
assert comparison.COMPILER_STYLES == {
|
||||
"raptor": ("#B3D5CA", "#4B7C6A"), "pimcomp": ("#E1BC8B", "#856A44")}
|
||||
assert comparison.METRIC_STYLES == {
|
||||
"latency": ("#B3CCE3", "#456B90"), "throughput": ("#B3D5CA", "#4B7C6A")}
|
||||
assert comparison.VALIDATION_STYLES["PASS"] == ("#B3D5CA", "#4B7C6A")
|
||||
assert comparison.TEXT_COLOR == "#222222"
|
||||
assert comparison.NETWORK_STYLES["yolo11n"] == ("#FF9494", "#C52020")
|
||||
assert comparison.NETWORK_STYLES["googlenet"] == ("#D1C0E0", "#77558F")
|
||||
rows = comparison.load_results(TOOLS / "pimcomp/compare/results_comparison.csv")
|
||||
selected = [replace(row, power=2 if row.arch == "arch-a" else 0.5)
|
||||
for row in rows if row.model == "vgg8" and row.mode == "latency"]
|
||||
captured = []
|
||||
monkeypatch.setattr(comparison, "save_figure", lambda fig, *args: captured.append(fig))
|
||||
comparison.draw_metric(selected, "power", "Average power advantage (x)", "power", tmp_path, diverging=True)
|
||||
comparison.draw_absolute(selected, "latency", "Single-inference latency (ms)", "latency", tmp_path)
|
||||
try:
|
||||
for ax, compiler in zip(captured[0].axes, ("raptor", "pimcomp")):
|
||||
bar = ax.containers[0].patches[0]
|
||||
assert bar.get_facecolor() == to_rgba(comparison.COMPILER_STYLES[compiler][0])
|
||||
assert bar.get_edgecolor() == to_rgba(comparison.COMPILER_STYLES[compiler][1])
|
||||
assert not bar.get_hatch()
|
||||
for ax in captured[1].axes:
|
||||
assert [container.patches[0].get_hatch() for container in ax.containers] == ["", ""]
|
||||
finally:
|
||||
for fig in captured:
|
||||
comparison.plt.close(fig)
|
||||
|
||||
|
||||
def test_tick_rotation_changes_only_overlapping_labels(comparison):
|
||||
fig, ax = comparison.plt.subplots(figsize=(4, 3), layout="constrained")
|
||||
try:
|
||||
ax.set_xlim(-1, 6)
|
||||
ax.set_xticks([0, 0.5, 5], ["Single-inference", "Throughput", "8"])
|
||||
comparison.fit_x_tick_labels(fig)
|
||||
assert [text.get_rotation() for text in ax.get_xticklabels()] == [330, 330, 0]
|
||||
assert [text.get_ha() for text in ax.get_xticklabels()] == ["left", "left", "center"]
|
||||
finally:
|
||||
comparison.plt.close(fig)
|
||||
|
||||
|
||||
def test_all_plot_terminology(comparison, monkeypatch):
|
||||
from matplotlib.path import Path as PlotPath
|
||||
from matplotlib.text import Text
|
||||
from matplotlib.transforms import Affine2D
|
||||
|
||||
plotter = comparison
|
||||
labels = set()
|
||||
|
||||
def capture(fig, name, output):
|
||||
fig.canvas.draw()
|
||||
if name != "raptor_pipeline_throughput":
|
||||
assert fig.axes[1].get_ylabel() == ""
|
||||
assert fig.axes[1].get_yticklabels() == []
|
||||
renderer = fig.canvas.get_renderer()
|
||||
for ax in fig.axes:
|
||||
for container in ax.containers:
|
||||
for bar in container.patches:
|
||||
center = bar.get_x() + bar.get_width() / 2
|
||||
bounds = bar.get_window_extent(renderer)
|
||||
for text in ax.texts:
|
||||
if hasattr(text, "xy") and text.xy[0] == center:
|
||||
extent = text.get_window_extent(renderer)
|
||||
assert bounds.x0 <= extent.x0 <= extent.x1 <= bounds.x1, (name, text.get_text())
|
||||
for legend in fig.legends:
|
||||
bounds = legend.get_window_extent(renderer)
|
||||
assert fig.bbox.x0 <= bounds.x0 <= bounds.x1 <= fig.bbox.x1, name
|
||||
tick_paths = []
|
||||
for ax in fig.axes:
|
||||
for minor in (False, True):
|
||||
for text in ax.get_xticklabels(minor=minor):
|
||||
if not text.get_visible() or not text.get_text():
|
||||
continue
|
||||
angle = text.get_rotation()
|
||||
assert angle in (0, 330), (name, text.get_text(), angle)
|
||||
bounds = text.get_window_extent(renderer)
|
||||
text.set_rotation(0)
|
||||
unrotated = text.get_window_extent(renderer)
|
||||
text.set_rotation(angle)
|
||||
path = PlotPath.unit_rectangle().transformed(
|
||||
Affine2D().scale(unrotated.width, unrotated.height).rotate_deg(angle))
|
||||
rotated = path.get_extents()
|
||||
path = path.transformed(Affine2D().translate(bounds.x0 - rotated.x0, bounds.y0 - rotated.y0))
|
||||
assert not any(path.intersects_path(other, filled=True) for other in tick_paths), name
|
||||
tick_paths.append(path)
|
||||
if text.get_text() in comparison.MODEL_NAMES.values():
|
||||
assert angle == 0, (name, text.get_text())
|
||||
heights = [ax.get_window_extent(renderer).height for ax in fig.axes]
|
||||
assert max(heights) - min(heights) < 1, name
|
||||
for ax in fig.axes:
|
||||
bounds = ax.get_window_extent(renderer)
|
||||
for text in ax.texts:
|
||||
extent = text.get_window_extent(renderer)
|
||||
assert bounds.y0 - 1 <= extent.y0 <= extent.y1 <= bounds.y1 + 1, (name, text.get_text())
|
||||
assert fig.bbox.x0 <= extent.x0 <= extent.x1 <= fig.bbox.x1, (name, text.get_text())
|
||||
labels.update(text.get_text() for text in fig.findobj(Text))
|
||||
plotter.plt.close(fig)
|
||||
|
||||
monkeypatch.setattr(plotter, "save_figure", capture)
|
||||
monkeypatch.setattr(sys, "argv", ["generate_plots.py"])
|
||||
plotter.main()
|
||||
text = "\n".join(labels)
|
||||
for old in ("Pimcomp", "ARCHITECTURE", "VGG8", "RESNET", "GOOGLENET", "YOLO11N",
|
||||
"Single inference", "Best case", "Latency, 1 stage", "Energy per inference advantage"):
|
||||
assert old not in text
|
||||
assert {"VGG-8", "ResNet-18", "ResNet-34", "GoogLeNet"} <= labels
|
||||
assert "Arch-A" in text and "Arch-B" in text
|
||||
assert {"Raptor", "PIMCOMP", "YOLO11n", "Single-inference", "2 stages", "4 stages", "8 stages",
|
||||
"Single-inference latency (ms)", "Throughput (inferences/s)", "Energy per inference (mJ)",
|
||||
"Energy-per-inference advantage (x)", "Derived as 1 / latency",
|
||||
"Throughput measurement"} <= labels
|
||||
|
||||
|
||||
@pytest.mark.parametrize("width,one_row", [(12, True), (4.5, False), (3, False)])
|
||||
def test_legend_uses_available_width_and_balances_rows(comparison, width, one_row):
|
||||
from collections import Counter
|
||||
from matplotlib.patches import Patch
|
||||
|
||||
fig = comparison.plt.figure(figsize=(width, 4), layout="constrained")
|
||||
try:
|
||||
handles = [Patch(label=f"Entry {index}") for index in range(7)]
|
||||
comparison.add_legend(fig, handles)
|
||||
fig.canvas.draw()
|
||||
renderer = fig.canvas.get_renderer()
|
||||
legend = fig.legends[0]
|
||||
counts = Counter(round(text.get_window_extent(renderer).y0, 3)
|
||||
for text in legend.get_texts() if text.get_visible())
|
||||
assert sum(counts.values()) == len(handles)
|
||||
assert (len(counts) == 1) == one_row
|
||||
assert max(counts.values()) - min(counts.values()) <= 1
|
||||
bounds = legend.get_window_extent(renderer)
|
||||
assert 0 <= bounds.x0 <= bounds.x1 <= fig.bbox.width
|
||||
finally:
|
||||
comparison.plt.close(fig)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("diverging", [False, True])
|
||||
def test_advantage_throughput_groups_touch(comparison, tmp_path, monkeypatch, diverging):
|
||||
rows = comparison.load_results(TOOLS / "pimcomp/compare/results_comparison.csv")
|
||||
selected = [row for row in rows if row.model in ("vgg8", "resnet18")]
|
||||
captured = []
|
||||
monkeypatch.setattr(comparison, "save_figure", lambda fig, *args: captured.append(fig))
|
||||
comparison.draw_metric(selected, "energy", "Energy-per-inference advantage (x)", "energy", tmp_path,
|
||||
diverging=diverging)
|
||||
fig = captured[0]
|
||||
try:
|
||||
for ax in fig.axes:
|
||||
bars = [container.patches[0] for container in ax.containers]
|
||||
lower, upper = ax.get_xlim()
|
||||
assert bars[0].get_x() - lower == pytest.approx(0.4)
|
||||
assert upper - bars[-1].get_x() - bars[-1].get_width() == pytest.approx(0.4)
|
||||
for index, (left, right) in enumerate(zip(bars, bars[1:])):
|
||||
edge = left.get_x() + left.get_width()
|
||||
if index % 4 in (1, 2):
|
||||
assert edge == pytest.approx(right.get_x())
|
||||
else:
|
||||
assert edge < right.get_x()
|
||||
finally:
|
||||
comparison.plt.close(fig)
|
||||
|
||||
|
||||
def test_grouped_absolute_plot_scheduling_labels(comparison, tmp_path, monkeypatch):
|
||||
rows = comparison.load_results(TOOLS / "pimcomp/compare/results_comparison.csv")
|
||||
captured = []
|
||||
monkeypatch.setattr(comparison, "save_figure", lambda fig, *args: captured.append(fig))
|
||||
comparison.draw_absolute(rows, "energy", "Energy per inference (mJ)", "energy", tmp_path, scale=1e-9)
|
||||
fig = captured[0]
|
||||
try:
|
||||
for ax in fig.axes:
|
||||
assert {text.get_text() for text in ax.get_xticklabels(minor=True)} == {"Latency", "Throughput"}
|
||||
labels = [text.get_text() for text in fig.legends[0].get_texts()]
|
||||
assert "1 stage (latency)" in labels and "Single-inference" not in labels
|
||||
finally:
|
||||
comparison.plt.close(fig)
|
||||
|
||||
|
||||
def test_pipeline_plot_connects_derived_and_measured_rates(comparison, tmp_path, monkeypatch):
|
||||
rows = comparison.load_results(TOOLS / "pimcomp/compare/results_comparison.csv")
|
||||
captured = []
|
||||
monkeypatch.setattr(comparison, "save_figure", lambda fig, *args: captured.append(fig))
|
||||
comparison.draw_pipeline_effect(rows, tmp_path)
|
||||
fig = captured[0]
|
||||
try:
|
||||
assert fig.get_size_inches() == pytest.approx((13, 4.3))
|
||||
assert fig._supylabel.get_text() == "Throughput (inferences/s)"
|
||||
for index, (ax, (arch, model)) in enumerate(zip(fig.axes, ((arch, model) for arch in comparison.ARCHITECTURES
|
||||
for model in comparison.MODELS))):
|
||||
selected = [row for row in rows if row.arch == arch and row.model == model]
|
||||
measured_line, derived_marker, *measured_markers = ax.lines
|
||||
assert measured_line.get_ydata()[0] == pytest.approx(1000 / selected[0].raptor_latency)
|
||||
assert list(measured_line.get_ydata()[1:]) == [row.raptor_throughput for row in selected[1:]]
|
||||
assert derived_marker.get_marker() == "s"
|
||||
assert derived_marker.get_ydata()[0] == pytest.approx(1000 / selected[0].raptor_latency)
|
||||
assert derived_marker.get_color() == comparison.NETWORK_STYLES[model][1]
|
||||
assert all(marker.get_marker() == "o" for marker in measured_markers)
|
||||
assert measured_line.get_color() == comparison.NETWORK_STYLES[model][1]
|
||||
assert [label.get_text() for label in ax.get_xticklabels()] == (
|
||||
["1", "2", "4", "8"] if arch == "arch-b" else [])
|
||||
assert ax.get_xlabel() == ""
|
||||
assert ax.get_title() == (comparison.MODEL_NAMES[model] if arch == "arch-a" else "")
|
||||
assert ax.title.get_weight() == "normal"
|
||||
if model == comparison.MODELS[-1]:
|
||||
assert ax.get_ylabel() == comparison.ARCH_NAMES[arch]
|
||||
assert ax.yaxis.get_label_position() == "right"
|
||||
assert ax.yaxis.label.get_rotation() == 270
|
||||
assert ax.yaxis.label.get_weight() == "bold"
|
||||
else:
|
||||
assert ax.get_ylabel() == ""
|
||||
assert all(label.get_rotation() == 0 for label in ax.get_xticklabels())
|
||||
lower, upper = ax.get_ylim()
|
||||
values = measured_line.get_ydata()
|
||||
assert 0 < lower < min(values) <= max(values) < upper
|
||||
assert ax.get_yticklabels()
|
||||
assert ax.get_ylim() == fig.axes[index % len(comparison.MODELS)].get_ylim()
|
||||
assert [text.get_text() for text in fig.legends[0].get_texts()] == [
|
||||
"Derived as 1 / latency", "Throughput measurement"]
|
||||
assert len({ax.lines[0].get_color() for ax in fig.axes[:len(comparison.MODELS)]}) == len(comparison.MODELS)
|
||||
finally:
|
||||
comparison.plt.close(fig)
|
||||
|
||||
|
||||
def test_plotter_exports_png_svg_and_pdf(comparison, tmp_path):
|
||||
comparison.configure_style()
|
||||
rows = comparison.load_results(TOOLS / "pimcomp/compare/results_comparison.csv")
|
||||
selected = [row for row in rows if row.model == "vgg8" and row.mode == "latency"]
|
||||
selected = [replace(row, raptor_validation="FAIL", pimcomp_latency=math.nan)
|
||||
for row in rows if row.model == "vgg8" and row.mode == "latency"]
|
||||
comparison.draw_absolute(selected, "latency", "Latency (ms)", "comparison-smoke", tmp_path)
|
||||
row = ablation.Result("baseline", "arch-a", "vgg8", "latency", 1, "PASS",
|
||||
{"latency": math.nan, "power": 0, "energy": 0, "throughput": math.nan})
|
||||
ablation.draw_metric([row], "latency", "Latency change (%)", tmp_path)
|
||||
for name in ("comparison-smoke", "latency_pipeline1_latency"):
|
||||
comparison.draw_validation(rows, tmp_path, compact=True)
|
||||
compact_names = []
|
||||
for metric in ("speedup", "energy", "power"):
|
||||
for suffix, diverging in (("comparison", False), ("diverging", True)):
|
||||
name = f"{metric}_compact_{suffix}"
|
||||
compact_names.append(name)
|
||||
comparison.draw_metric(comparison.select_compact_speedups(rows), metric, "Advantage (x)",
|
||||
name, tmp_path, diverging=diverging, compact=True)
|
||||
for name in ("comparison-smoke", "validation_status_compact", *compact_names):
|
||||
assert (tmp_path / f"png/{name}.png").read_bytes().startswith(b"\x89PNG\r\n\x1a\n")
|
||||
svg = (tmp_path / f"svg/{name}.svg").read_text()
|
||||
assert "<svg" in svg
|
||||
assert "N/A" in (tmp_path / "svg/latency_pipeline1_latency.svg").read_text()
|
||||
assert (tmp_path / f"pdf/{name}.pdf").read_bytes().startswith(b"%PDF-")
|
||||
assert "N/A" in (tmp_path / "svg/comparison-smoke.svg").read_text()
|
||||
assert "P. INVALID" in (tmp_path / "svg/comparison-smoke.svg").read_text()
|
||||
|
||||
@@ -47,27 +47,9 @@ guarantees, so those variants may fail numerical validation and exit nonzero.
|
||||
Baseline and `no-plan` must pass. Percentages from failing variants are not
|
||||
valid speedups; deadline averages do not prove steady-state convergence.
|
||||
|
||||
## Plots
|
||||
## Result data
|
||||
|
||||
```bash
|
||||
.venv/bin/python validation/tools/pim/ablation/generate_plots.py
|
||||
# For a custom experiment root:
|
||||
.venv/bin/python validation/tools/pim/ablation/generate_plots.py \
|
||||
--input /path/to/experiment/results_ablation.csv
|
||||
```
|
||||
|
||||
Requires Matplotlib from the root `requirements.txt`. By default, reads the
|
||||
canonical CSV beside this script. Plots go to
|
||||
`plots/png/` and `plots/svg/` beside the selected CSV; override with
|
||||
`--output-dir PATH`.
|
||||
Single-variant comparison CSVs are not accepted.
|
||||
|
||||
The checked-in CSV is an intentional result dataset; plotting uses its current
|
||||
contents without running benchmarks. Invalid reciprocal percentage pairs render
|
||||
as `N/A`; individual legitimate extreme changes are preserved. See
|
||||
[performance availability](../../../../docs/evaluation/methodology.md#functional-validity-and-performance-availability).
|
||||
|
||||
Plots separate modes and pipelines. Throughput-mode latency means output
|
||||
interval. Failed validation is labeled `INVALID`, other non-PASS statuses keep
|
||||
their names, and missing measurements show `N/A`. See the
|
||||
The checked-in `results_ablation.csv` is an intentional result dataset for tabular
|
||||
reporting. Ablation experiments and aggregation remain available through the
|
||||
runner above. Percentages from failing variants are not valid speedups. See the
|
||||
[comparison metric definitions](../../../../docs/evaluation/methodology.md#metric-interpretation).
|
||||
|
||||
@@ -1,156 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Plot percentage changes from an experiment's combined results_ablation.csv."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import math
|
||||
import sys
|
||||
import warnings
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
|
||||
from plot_style import COLORS, configure_style, save_figure, value_label
|
||||
|
||||
import matplotlib.pyplot as plt
|
||||
from matplotlib.patches import Patch
|
||||
from matplotlib.ticker import PercentFormatter
|
||||
|
||||
from run_ablation import ABLATION_DEFAULT_MODELS, DEFAULT_RESULTS_PATH, RESULT_FIELDS, VARIANTS
|
||||
|
||||
|
||||
VARIANT_COLORS = {variant.name: color for variant, color in zip(VARIANTS, COLORS)}
|
||||
STATUSES = ("PASS", "FAIL", "SKIPPED", "NA")
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class Result:
|
||||
variant: str
|
||||
arch: str
|
||||
model: str
|
||||
mode: str
|
||||
pipeline: int
|
||||
validation: str
|
||||
metrics: dict[str, float]
|
||||
|
||||
|
||||
def percentage(value: str) -> float:
|
||||
if value == "NA":
|
||||
return math.nan
|
||||
if not value.endswith("%"):
|
||||
raise ValueError(f"Expected a percentage or NA, got {value!r}")
|
||||
parsed = float(value[:-1])
|
||||
if not math.isfinite(parsed):
|
||||
raise ValueError(f"Non-finite percentage: {value!r}")
|
||||
return parsed
|
||||
|
||||
|
||||
def load_results(path: Path) -> list[Result]:
|
||||
rows = []
|
||||
seen = set()
|
||||
invalid_pairs = []
|
||||
with path.open(newline="", encoding="utf-8-sig") as stream:
|
||||
reader = csv.DictReader(stream)
|
||||
if not {"variant", *RESULT_FIELDS} <= set(reader.fieldnames or []):
|
||||
raise ValueError("Expected the combined ablation CSV with variant and percentage columns; "
|
||||
"a per-variant results_comparison.csv cannot be used here.")
|
||||
for row in reader:
|
||||
variant, mode, status = row["variant"], row["mode"], row["functional_validation"]
|
||||
if variant not in VARIANT_COLORS or mode not in ("latency", "throughput") or status not in STATUSES:
|
||||
raise ValueError(f"Unknown ablation variant, mode, or validation status: {variant}, {mode}, {status}")
|
||||
pipeline = int(row["raptor_pipeline"])
|
||||
key = (variant, row["arch"], row["model"], mode, pipeline)
|
||||
if key in seen:
|
||||
raise ValueError(f"Duplicate ablation configuration: {key}")
|
||||
seen.add(key)
|
||||
metrics = {metric: percentage(row[metric])
|
||||
for metric in ("latency", "throughput", "power", "energy")}
|
||||
# Each may round to -100% alone; reciprocal metrics cannot both do so.
|
||||
if mode == "throughput" and metrics["latency"] == metrics["throughput"] == -100:
|
||||
metrics["latency"] = metrics["throughput"] = math.nan
|
||||
invalid_pairs.append(key)
|
||||
rows.append(Result(variant, row["arch"], row["model"], mode, pipeline, status, metrics))
|
||||
if not rows:
|
||||
raise ValueError("The ablation CSV contains no results")
|
||||
if invalid_pairs:
|
||||
warnings.warn(f"{len(invalid_pairs)} invalid reciprocal latency/throughput percentage pairs; "
|
||||
f"displaying N/A. Examples: {invalid_pairs[:3]}", stacklevel=2)
|
||||
return rows
|
||||
|
||||
|
||||
def draw_metric(results: list[Result], metric: str, ylabel: str, output_dir: Path) -> None:
|
||||
architectures = sorted({row.arch for row in results})
|
||||
present_models = {row.model for row in results}
|
||||
models = [model for model in ABLATION_DEFAULT_MODELS if model in present_models]
|
||||
models += sorted(present_models - set(models))
|
||||
variants = [name for name in VARIANT_COLORS if any(row.variant == name for row in results)]
|
||||
lookup = {(row.arch, row.model, row.variant): row for row in results}
|
||||
values = [row.metrics[metric] for row in results if math.isfinite(row.metrics[metric])]
|
||||
lower, upper = min(0, min(values, default=0)), max(0, max(values, default=0))
|
||||
padding = max(upper - lower, 1) * 0.35
|
||||
fig, axes = plt.subplots(1, len(architectures), sharey=True, squeeze=False,
|
||||
figsize=(7 * len(architectures), 5.2), layout="constrained")
|
||||
handles = [Patch(facecolor=VARIANT_COLORS[name], label=name) for name in variants]
|
||||
if len(values) < len(architectures) * len(models) * len(variants):
|
||||
handles.append(Patch(facecolor="white", edgecolor="#B2BAC5", label="N/A: missing measurement"))
|
||||
fig.legend(handles=handles, loc="outside upper center", ncol=4, frameon=False, fontsize=9.5)
|
||||
width = 0.8 / len(variants)
|
||||
mode, pipeline = results[0].mode, results[0].pipeline
|
||||
for ax, arch in zip(axes[0], architectures):
|
||||
ax.set_axisbelow(True)
|
||||
ax.set_ylim(lower - padding if lower < 0 else 0, upper + padding)
|
||||
ax.set_xlim(-0.5, len(models) - 0.5)
|
||||
ax.set_xticks(range(len(models)), [model.upper() for model in models], fontsize=9.5)
|
||||
ax.set_xlabel(f"{arch.replace('arch-', 'ARCHITECTURE ').upper()}\n"
|
||||
f"{mode.capitalize()}, {pipeline} stage" + ("s" if pipeline != 1 else ""),
|
||||
fontsize=11, weight="bold", labelpad=8)
|
||||
ax.set_ylabel(ylabel, fontsize=10.5)
|
||||
ax.yaxis.set_major_formatter(PercentFormatter(xmax=100, decimals=0))
|
||||
ax.tick_params(axis="both", length=0, pad=6, labelleft=True)
|
||||
ax.grid(axis="y", color="#E6EAF0", linewidth=0.8)
|
||||
ax.axhline(0, color="#536478", linewidth=1.2, linestyle="--")
|
||||
for index, model in enumerate(models):
|
||||
if index % 2 == 0:
|
||||
ax.axvspan(index - 0.5, index + 0.5, facecolor="#F7F9FC", zorder=0)
|
||||
for offset, variant in enumerate(variants):
|
||||
x = index + (offset - (len(variants) - 1) / 2) * width
|
||||
row = lookup.get((arch, model, variant))
|
||||
value = row.metrics[metric] if row else math.nan
|
||||
if not math.isfinite(value):
|
||||
ax.text(x, padding * 0.05, "N/A", rotation=90, ha="center", va="bottom",
|
||||
fontsize=8, color="#85909F")
|
||||
continue
|
||||
ax.bar(x, value, width=width * 0.9, color=VARIANT_COLORS[variant],
|
||||
edgecolor="none", linewidth=0, zorder=2)
|
||||
direction = -1 if value < 0 else 1
|
||||
value_label(ax, x, value + direction * padding * 0.05, f"{value:+,.2f}%",
|
||||
row.validation, direction)
|
||||
save_figure(fig, f"{mode}_pipeline{pipeline}_{metric}", output_dir)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description=__doc__)
|
||||
parser.add_argument("--input", type=Path, default=DEFAULT_RESULTS_PATH)
|
||||
parser.add_argument("--output-dir", type=Path)
|
||||
args = parser.parse_args()
|
||||
if args.output_dir is None:
|
||||
args.output_dir = args.input.parent / "plots"
|
||||
configure_style()
|
||||
results = load_results(args.input)
|
||||
print(f"Results: {len(results)} configurations, "
|
||||
f"{sum(row.validation != 'PASS' for row in results)} without PASS validation")
|
||||
for mode, pipeline in sorted({(row.mode, row.pipeline) for row in results}):
|
||||
rows = [row for row in results if row.mode == mode and row.pipeline == pipeline]
|
||||
for metric, label in (
|
||||
("latency", "Single-inference latency" if mode == "latency" else "Output interval"),
|
||||
("throughput", "Throughput"), ("power", "Average power"), ("energy", "Energy per inference"),
|
||||
):
|
||||
if metric == "throughput" and mode == "latency":
|
||||
continue
|
||||
draw_metric(rows, metric, f"{label} change from baseline (%)", args.output_dir)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -72,7 +72,7 @@ The comparator defaults to absolute/relative tolerances `1e-3`/`1e-4`;
|
||||
|
||||
Requires Matplotlib from the root `requirements.txt`. Defaults to the supplied
|
||||
`results_comparison.csv` beside the script and writes `plots/png/` and
|
||||
`plots/svg/` there. Use `--input PATH` for a runner-generated CSV and
|
||||
`plots/svg/` and `plots/pdf/` there. Use `--input PATH` for a runner-generated CSV and
|
||||
`--output-dir PATH` to override the destination. The plotter requires the full
|
||||
40-case matrix: five models × Arch-A/B × latency pipeline 1 and throughput
|
||||
pipelines 2/4/8. Duplicate, missing, unexpected, or inconsistent shared Pimcomp
|
||||
@@ -81,6 +81,9 @@ configurations are rejected before plots are written.
|
||||
| Plot family | Meaning |
|
||||
|---|---|
|
||||
| `validation_status` | Recorded functional status for each compiler and experiment. |
|
||||
| `validation_status_compact` | Compact functional status matrix. |
|
||||
| `speedup_compact_*` | Touching single-inference latency and best-pipeline throughput bars for each network. |
|
||||
| `energy_compact_*`, `power_compact_*` | Energy and power advantages for the same cases selected by the compact speedup plots. |
|
||||
| `latency_*` | Single-inference latency, latency mode only. |
|
||||
| `throughput_*` | Inferences/s, throughput mode only. |
|
||||
| `longest_stage_latency_*` | Combined single-inference latency and throughput output interval; the latter is a stage-latency proxy. |
|
||||
@@ -93,7 +96,28 @@ advantages in either direction. Ratios above 1 favor Raptor: Pimcomp/Raptor for
|
||||
latency, power, and energy; Raptor/Pimcomp for throughput. Diverging bars use
|
||||
winning factor minus one, with parity at zero.
|
||||
|
||||
Failed validation is labeled `INVALID`; other non-PASS statuses keep their
|
||||
The shared `plot_style.py` defines the manuscript palette and display names.
|
||||
Raptor uses sage green; PIMCOMP uses plain sand. Stage patterns,
|
||||
outlines, marker shapes, and status text preserve distinctions in grayscale.
|
||||
Compact plots use blue for the single-inference case and hatched sage
|
||||
green for the throughput case.
|
||||
Each architecture/network pair uses the highest recorded throughput across pipelines.
|
||||
Energy and power use those same cases, without selecting a different pipeline for each metric.
|
||||
Validation labels remain visible for the selected cases. Units appear in axis
|
||||
titles and bar values; axis tick labels contain numbers only.
|
||||
|
||||
Throughput-stage bars touch within each network in both absolute and advantage
|
||||
plots. Bar-label widths are measured before export; dense figures expand only
|
||||
as needed to contain values and invalidity annotations. Legends use the available
|
||||
width and balance entries across rows when wrapping is necessary.
|
||||
|
||||
Comparison panels share y-axis titles and tick labels. Each network in the
|
||||
pipeline plot retains its own scale, with y-axis ticks on both architecture rows.
|
||||
Only the bottom row shows the shared x-axis ticks. Network names appear above
|
||||
the top row; architecture names appear on the right. That plot
|
||||
trims unused vertical range and is intended to span both manuscript columns.
|
||||
|
||||
Failed validation is labeled `P. INVALID`; other non-PASS statuses keep their
|
||||
names, and missing measurements show `N/A`. Relative bars require both
|
||||
compilers to pass to be valid. Repeated Pimcomp batch measurements and statuses
|
||||
must agree before being collapsed. `FAIL` alone does not identify the cause.
|
||||
|
||||
@@ -1,25 +1,25 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate publication-ready Raptor vs Pimcomp comparison charts.
|
||||
"""Generate publication-ready Raptor vs PIMCOMP comparison charts.
|
||||
|
||||
Usage:
|
||||
.venv/bin/python validation/tools/pim/pimcomp/compare/generate_plots.py
|
||||
|
||||
Dependencies: matplotlib (no pandas or seaborn required).
|
||||
By default, reads results_comparison.csv beside this script and writes plots/
|
||||
beside it, with outputs grouped in png/ and svg/ under the output directory.
|
||||
beside it, with outputs grouped in png/, svg/, and pdf/ under the output directory.
|
||||
|
||||
YOLO models without Pimcomp measurements get separate Raptor-only charts.
|
||||
YOLO models without PIMCOMP measurements get separate Raptor-only charts.
|
||||
Comparisons use vertical bars for ratios, diverging advantages, and absolute
|
||||
values, with compiler colors and patterns for the Raptor pipeline stages.
|
||||
Throughput-mode latency is the output interval, a bottleneck-stage proxy,
|
||||
not end-to-end inference latency. Separate throughput plots use inferences/s.
|
||||
Energy is reported per inference; power is averaged over the simulation.
|
||||
Numerical results whose compiler did not pass functional validation have a red
|
||||
status following their value label; missing Pimcomp
|
||||
status following their value label; missing PIMCOMP
|
||||
measurements appear as 'N/A'. The fixed compiler palette is shared by
|
||||
all metrics. Relative plots use Pimcomp / Raptor for latency, power and energy,
|
||||
and Raptor / Pimcomp for throughput, so >1 favors Raptor. Diverging plots put the winner's
|
||||
factor above (Raptor) or below (Pimcomp) parity, with 1x at the zero line.
|
||||
all metrics. Relative plots use PIMCOMP / Raptor for latency, power and energy,
|
||||
and Raptor / PIMCOMP for throughput, so >1 favors Raptor. Diverging plots put the winner's
|
||||
factor above (Raptor) or below (PIMCOMP) parity, with 1x at the zero line.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
@@ -39,20 +39,22 @@ from matplotlib.lines import Line2D
|
||||
from matplotlib.ticker import FuncFormatter, MultipleLocator
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parents[2]))
|
||||
from plot_style import COLORS, HATCH_COLOR, TEXT_COLOR, configure_style, fit_value_labels, save_figure, value_label
|
||||
from plot_style import (ARCH_NAMES, BAND_COLOR, COMPILER_NAMES, COMPILER_STYLES,
|
||||
ERROR_COLOR, LINE_COLOR, MATRIX_TEXT_COLOR, METRIC_STYLES,
|
||||
MODE_NAMES, MODEL_NAMES, NETWORK_STYLES,
|
||||
PALETTE, TEXT_COLOR, add_legend, configure_style, fit_value_labels,
|
||||
fit_bar_widths, fit_x_tick_labels, save_figure, value_label)
|
||||
|
||||
|
||||
COMPILER_COLORS = {"raptor": COLORS[0], "pimcomp": COLORS[2]}
|
||||
STAGE_PATTERNS = {1: "", 2: "..", 4: "//", 8: "\\\\"}
|
||||
ARCHITECTURES = ("arch-a", "arch-b")
|
||||
MODELS = ("vgg8", "resnet18", "resnet34", "googlenet", "yolo11n")
|
||||
CONFIGURATIONS = (("latency", 1), ("throughput", 2), ("throughput", 4), ("throughput", 8))
|
||||
LINE_COLOR = "#A8B2C0"
|
||||
VALIDATION_STYLES = {
|
||||
"PASS": ("#168C8B", "white", ""),
|
||||
"FAIL": ("#D18D32", HATCH_COLOR, ""),
|
||||
"SKIPPED": ("#E7DFF0", TEXT_COLOR, ""),
|
||||
"NA": ("#E8EDF1", TEXT_COLOR, ""),
|
||||
"PASS": PALETTE["sage"],
|
||||
"FAIL": PALETTE["red"],
|
||||
"SKIPPED": PALETTE["pale_sand"],
|
||||
"NA": PALETTE["pale_blue"],
|
||||
}
|
||||
|
||||
|
||||
@@ -85,6 +87,10 @@ class Result:
|
||||
def throughput_speedup(self) -> float:
|
||||
return quotient(self.raptor_throughput, self.pimcomp_throughput)
|
||||
|
||||
@property
|
||||
def speedup(self) -> float:
|
||||
return self.latency_speedup if self.mode == "latency" else self.throughput_speedup
|
||||
|
||||
|
||||
def positive_float(text: str | None) -> float:
|
||||
try:
|
||||
@@ -129,7 +135,7 @@ def load_results(path: Path) -> list[Result]:
|
||||
seen[key] = line
|
||||
schedule = row["pimcomp_pipeline"].strip().lower()
|
||||
if schedule != ("element" if mode == "latency" else "batch"):
|
||||
raise ValueError(f"{key}: incompatible Pimcomp schedule {schedule!r}")
|
||||
raise ValueError(f"{key}: incompatible PIMCOMP schedule {schedule!r}")
|
||||
rows.append(Result(
|
||||
arch=arch,
|
||||
model=model,
|
||||
@@ -185,20 +191,20 @@ def figure_legend(fig, results: list[Result], field: str,
|
||||
for _, compiler, value in available} - {None}
|
||||
if len(visible_compilers) > 1:
|
||||
for compiler in compilers:
|
||||
handles.append(Patch(facecolor=COMPILER_COLORS[compiler],
|
||||
label=compiler.capitalize() + ("" if absolute else " advantage")))
|
||||
fill, border = COMPILER_STYLES[compiler]
|
||||
handles.append(Patch(facecolor=fill, edgecolor=border,
|
||||
label=COMPILER_NAMES[compiler] + ("" if absolute else " advantage")))
|
||||
handles += [Patch(facecolor="white", edgecolor=TEXT_COLOR, hatch=STAGE_PATTERNS[depth],
|
||||
label=f"{depth} stage" + (" (latency)" if depth == 1 else "s"))
|
||||
label=("1 stage (latency)" if absolute else "Single-inference")
|
||||
if depth == 1 else f"{depth} stages")
|
||||
for depth in sorted(stages) if stages != {1}]
|
||||
styles = []
|
||||
if len(available) < len(measurements):
|
||||
styles.append(Patch(facecolor="white", edgecolor="#B2BAC5",
|
||||
styles.append(Patch(facecolor="white", edgecolor=LINE_COLOR,
|
||||
label="N/A: missing measurement"))
|
||||
handles += styles
|
||||
if handles:
|
||||
fig.legend(handles=handles, loc="outside upper center", ncol=(len(handles) + 1) // 2,
|
||||
frameon=False, fontsize=9.5, handlelength=1.7, columnspacing=1.5,
|
||||
handletextpad=0.6, labelspacing=0.4)
|
||||
add_legend(fig, handles)
|
||||
|
||||
|
||||
def absolute_columns(results: list[Result], raptor_only: bool = False) -> list[tuple[Result, str]]:
|
||||
@@ -218,7 +224,7 @@ def absolute_columns(results: list[Result], raptor_only: bool = False) -> list[t
|
||||
for row in throughput
|
||||
}
|
||||
if len(signatures) != 1:
|
||||
raise ValueError(f"{arch}/{model}: inconsistent shared Pimcomp batch results across pipelines")
|
||||
raise ValueError(f"{arch}/{model}: inconsistent shared PIMCOMP batch results across pipelines")
|
||||
columns.append((throughput[0], "pimcomp"))
|
||||
return columns
|
||||
|
||||
@@ -231,34 +237,50 @@ def label_groups(ax, results: list[Result], positions: list[float] | None = None
|
||||
for index, (model, rows) in enumerate(groupby(results, key=lambda row: row.model)):
|
||||
end = start + len(list(rows))
|
||||
centers.append((positions[start] + positions[end - 1]) / 2)
|
||||
labels.append(model.upper())
|
||||
labels.append(MODEL_NAMES[model])
|
||||
left = ax.get_xlim()[0] if start == 0 else (positions[start - 1] + positions[start]) / 2
|
||||
right = ax.get_xlim()[1] if end == len(results) else (positions[end - 1] + positions[end]) / 2
|
||||
if index % 2 == 0:
|
||||
ax.axvspan(left, right, facecolor="#F7F9FC", zorder=0)
|
||||
ax.axvspan(left, right, facecolor=BAND_COLOR, zorder=0)
|
||||
if index > 0:
|
||||
ax.axvline(left, color="#DDE3EB", linewidth=0.9)
|
||||
ax.axvline(left, color=LINE_COLOR, linewidth=0.7)
|
||||
start = end
|
||||
ax.set_xticks(centers, labels, fontsize=9.5)
|
||||
|
||||
|
||||
def grouped_positions(results: list[Result]) -> list[float]:
|
||||
positions = []
|
||||
for index, row in enumerate(results):
|
||||
if index == 0:
|
||||
positions.append(0.0)
|
||||
continue
|
||||
previous = results[index - 1]
|
||||
gap = 0.8 if row.model != previous.model else 0.4 if row.mode != previous.mode else 0.0
|
||||
positions.append(positions[-1] + 1.0 + gap)
|
||||
return positions
|
||||
|
||||
|
||||
def diverging_ratio(ratio: float) -> float:
|
||||
return ratio - 1 if ratio >= 1 else 1 - 1 / ratio
|
||||
|
||||
|
||||
def draw_validation(results: list[Result], output_dir: Path) -> None:
|
||||
columns = absolute_columns(results)
|
||||
configurations = list(dict.fromkeys((row.mode, row.pipeline if compiler == "raptor" else row.pimcomp_schedule,
|
||||
compiler) for row, compiler in columns))
|
||||
def draw_validation(results: list[Result], output_dir: Path, compact: bool = False) -> None:
|
||||
cells = {}
|
||||
for row, compiler in absolute_columns(results):
|
||||
stage = row.pipeline if compiler == "raptor" else row.pimcomp_schedule
|
||||
key = (row.arch, row.model, row.mode, None if compact else stage, compiler)
|
||||
cells.setdefault(key, set()).add(getattr(row, f"{compiler}_validation"))
|
||||
configurations = list(dict.fromkeys(key[2:] for key in cells))
|
||||
positions = []
|
||||
for index, (mode, _, _) in enumerate(configurations):
|
||||
gap = 0.4 if index and mode != configurations[index - 1][0] else 0
|
||||
positions.append(positions[-1] + 1 + gap if index else 0)
|
||||
configuration_x = dict(zip(configurations, positions))
|
||||
headers = [f"Raptor\n{stage} stages" if mode == "throughput" and compiler == "raptor"
|
||||
else "Pimcomp\nbatch" if mode == "throughput" else compiler.capitalize()
|
||||
headers = [f"Raptor\n{stage} stages" if mode == "throughput" and compiler == "raptor" and not compact
|
||||
else "PIMCOMP\nbatch" if mode == "throughput" and compiler == "pimcomp" and not compact
|
||||
else COMPILER_NAMES[compiler]
|
||||
for mode, stage, compiler in configurations]
|
||||
fig, axes = plt.subplots(1, 2, figsize=(14, 3.8), layout="constrained")
|
||||
fig, axes = plt.subplots(1, 2, figsize=(8.5 if compact else 12, 3.3), layout="constrained")
|
||||
for ax, arch in zip(axes, ARCHITECTURES):
|
||||
ax.set_xlim(positions[0] - 0.55, positions[-1] + 0.55)
|
||||
ax.set_ylim(len(MODELS) - 0.5, -0.5)
|
||||
@@ -266,31 +288,41 @@ def draw_validation(results: list[Result], output_dir: Path) -> None:
|
||||
for mode, group in groupby(zip(configurations, positions), key=lambda item: item[0][0]):
|
||||
group_positions = [x for _, x in group]
|
||||
centers.append((group_positions[0] + group_positions[-1]) / 2)
|
||||
modes.append(mode.capitalize())
|
||||
modes.append("Latency" if mode == "latency" else MODE_NAMES[mode])
|
||||
ax.set_xticks(centers, modes, fontsize=10)
|
||||
ax.set_xticks(positions, headers, minor=True)
|
||||
ax.tick_params(axis="x", which="minor", top=True, labeltop=True,
|
||||
bottom=False, labelbottom=False, length=0, pad=6, labelsize=9)
|
||||
ax.set_yticks(range(len(MODELS)), [model.upper() for model in MODELS], fontsize=9)
|
||||
ax.set_yticks(range(len(MODELS)), [MODEL_NAMES[model] for model in MODELS], fontsize=9)
|
||||
ax.tick_params(axis="both", length=0, pad=6)
|
||||
architecture = "ARCHITECTURE A" if arch == "arch-a" else "ARCHITECTURE B"
|
||||
ax.tick_params(axis="y", labelleft=arch == ARCHITECTURES[0])
|
||||
architecture = ARCH_NAMES[arch]
|
||||
ax.set_xlabel(architecture, fontsize=11, weight="bold", labelpad=8)
|
||||
for row, compiler in columns:
|
||||
if row.arch != arch:
|
||||
for (cell_arch, model, mode, stage, compiler), statuses in cells.items():
|
||||
if cell_arch != arch:
|
||||
continue
|
||||
key = (row.mode, row.pipeline if compiler == "raptor" else row.pimcomp_schedule, compiler)
|
||||
x, y = configuration_x[key], MODELS.index(row.model)
|
||||
status = getattr(row, f"{compiler}_validation")
|
||||
color, text_color, _ = VALIDATION_STYLES[status]
|
||||
x, y = configuration_x[(mode, stage, compiler)], MODELS.index(model)
|
||||
status = next(status for status in ("FAIL", "SKIPPED", "NA", "PASS") if status in statuses)
|
||||
color, border = VALIDATION_STYLES[status]
|
||||
ax.add_patch(Rectangle((x - 0.5, y - 0.42), 1, 0.84,
|
||||
facecolor=color, edgecolor="#657081", linewidth=0.5))
|
||||
facecolor=color, edgecolor=border, linewidth=0.7))
|
||||
ax.text(x, y, "N/A" if status == "NA" else status,
|
||||
color=text_color, ha="center", va="center", fontsize=9)
|
||||
save_figure(fig, "validation_status", output_dir)
|
||||
color=MATRIX_TEXT_COLOR, ha="center", va="center", fontsize=12, weight="normal")
|
||||
fit_x_tick_labels(fig)
|
||||
save_figure(fig, "validation_status_compact" if compact else "validation_status", output_dir)
|
||||
|
||||
|
||||
def select_compact_speedups(results: list[Result]) -> list[Result]:
|
||||
selected = []
|
||||
for _, group in groupby(results, key=lambda row: (row.arch, row.model, row.mode)):
|
||||
rows = list(group)
|
||||
selected.append(max(rows, key=lambda row: row.speedup if math.isfinite(row.speedup) else -math.inf))
|
||||
return selected
|
||||
|
||||
|
||||
def draw_metric(results: list[Result], field: str, ylabel: str,
|
||||
basename: str, output_dir: Path, diverging: bool = False) -> None:
|
||||
basename: str, output_dir: Path, diverging: bool = False,
|
||||
compact: bool = False) -> None:
|
||||
data = [diverging_ratio(getattr(row, field)) if diverging else getattr(row, field)
|
||||
for row in results]
|
||||
available = [v for v in data if math.isfinite(v)]
|
||||
@@ -301,49 +333,68 @@ def draw_metric(results: list[Result], field: str, ylabel: str,
|
||||
y_min = math.floor((lower - padding) * 2) / 2 if diverging else 0
|
||||
y_interval = 1.0 if y_max - y_min > 4.5 else 0.5
|
||||
|
||||
fig, axes = plt.subplots(1, 2, sharey=True, figsize=(14, 5.2),
|
||||
fig, axes = plt.subplots(1, 2, sharey=True, figsize=(8.5 if compact else 12, 4.4),
|
||||
layout="constrained")
|
||||
figure_legend(fig, results, field)
|
||||
if compact:
|
||||
single_metric = "latency" if field == "speedup" else field
|
||||
throughput_suffix = "" if field == "speedup" else f" {field}"
|
||||
handles = [Patch(facecolor=METRIC_STYLES["latency"][0],
|
||||
edgecolor=METRIC_STYLES["latency"][1],
|
||||
label=f"Single-inference {single_metric}"),
|
||||
Patch(facecolor=METRIC_STYLES["throughput"][0],
|
||||
edgecolor=METRIC_STYLES["throughput"][1], hatch="//",
|
||||
label=f"Best-case throughput{throughput_suffix}")]
|
||||
|
||||
for ax, arch in zip(axes, ARCHITECTURES):
|
||||
selected = [row for row in results if row.arch == arch]
|
||||
ax.set_xlabel("ARCHITECTURE A" if arch == "arch-a" else "ARCHITECTURE B",
|
||||
positions = [index + index // 2 for index in range(len(selected))] if compact else grouped_positions(selected)
|
||||
ax.set_xlabel(ARCH_NAMES[arch],
|
||||
fontsize=11, weight="bold", labelpad=8)
|
||||
ax.set_axisbelow(True)
|
||||
ax.set_ylim(y_min, y_max)
|
||||
ax.set_xlim(-0.5, len(selected) - 0.5)
|
||||
label_groups(ax, selected)
|
||||
ax.tick_params(axis="y", labelleft=True, pad=5, length=0)
|
||||
ax.set_xlim(-0.9, positions[-1] + 0.9 if positions else 0.9)
|
||||
label_groups(ax, selected, positions)
|
||||
ax.tick_params(axis="y", labelleft=arch == ARCHITECTURES[0], pad=5, length=0)
|
||||
ax.tick_params(axis="x", labelsize=9.5, length=0, pad=6)
|
||||
ax.yaxis.set_major_locator(MultipleLocator(y_interval))
|
||||
if diverging:
|
||||
ax.yaxis.set_major_formatter(FuncFormatter(lambda value, _: f"{abs(value) + 1:g}×"))
|
||||
ax.grid(axis="y", color="#E6EAF0", linewidth=0.8)
|
||||
ax.axhline(0 if diverging else 1, color="#536478", linewidth=1.2,
|
||||
ax.yaxis.set_major_formatter(FuncFormatter(lambda value, _: f"{abs(value) + 1:g}"))
|
||||
ax.grid(axis="y")
|
||||
ax.axhline(0 if diverging else 1, color=TEXT_COLOR, linewidth=1.0,
|
||||
linestyle="--", zorder=3)
|
||||
ax.set_ylabel(ylabel, fontsize=10.5, labelpad=6)
|
||||
if arch == ARCHITECTURES[0]:
|
||||
ax.set_ylabel(ylabel, fontsize=10.5, labelpad=6)
|
||||
|
||||
for position, row in enumerate(selected):
|
||||
for position, row in zip(positions, selected):
|
||||
ratio = getattr(row, field)
|
||||
if not math.isfinite(ratio):
|
||||
ax.text(position, (y_max - y_min) * 0.02, "N/A", ha="center", va="bottom",
|
||||
fontsize=9.5, color="#85909F", fontstyle="italic")
|
||||
fontsize=9.5, color=TEXT_COLOR, fontstyle="italic")
|
||||
continue
|
||||
compiler = advantage_compiler(ratio)
|
||||
statuses = [status for status in (row.raptor_validation, row.pimcomp_validation) if status != "PASS"]
|
||||
status = "FAIL" if "FAIL" in statuses else next(iter(statuses), "PASS")
|
||||
pattern = STAGE_PATTERNS[row.pipeline]
|
||||
color, border = COMPILER_STYLES[compiler] if compiler else ("white", TEXT_COLOR)
|
||||
if compact:
|
||||
pattern = "//" if row.mode == "throughput" else ""
|
||||
color, border = METRIC_STYLES[row.mode]
|
||||
measurement = diverging_ratio(ratio) if diverging else ratio
|
||||
ax.bar(position, measurement, width=0.72,
|
||||
color=COMPILER_COLORS.get(compiler, "#657081"),
|
||||
ax.bar(position, measurement, width=1.0,
|
||||
color=color,
|
||||
hatch=pattern,
|
||||
edgecolor=TEXT_COLOR if pattern else "none",
|
||||
linewidth=0.7 if pattern else 0.0, zorder=2)
|
||||
edgecolor=border, linewidth=0.7, zorder=2)
|
||||
direction = -1 if measurement < 0 else 1
|
||||
factor = max(ratio, 1 / ratio) if diverging else ratio
|
||||
value_label(ax, position, measurement + direction * (y_max - y_min) * 0.012,
|
||||
f"{factor:.2f}×", status, direction, fontsize=9.1)
|
||||
f"{factor:.2f}×", status, direction, fontsize=9.1, invalid_label="P. INVALID")
|
||||
|
||||
fit_bar_widths(fig)
|
||||
if compact:
|
||||
add_legend(fig, handles)
|
||||
else:
|
||||
figure_legend(fig, results, field)
|
||||
fit_x_tick_labels(fig)
|
||||
fit_value_labels(fig)
|
||||
save_figure(fig, basename, output_dir)
|
||||
|
||||
@@ -356,21 +407,13 @@ def draw_absolute(results: list[Result], field: str, ylabel: str,
|
||||
for row, compiler in columns
|
||||
if math.isfinite(getattr(row, f"{compiler}_{field}"))]
|
||||
y_max = max(available, default=1.0) * 1.35
|
||||
fig, axes = plt.subplots(1, 2, sharey=True, figsize=(14, 5.2),
|
||||
fig, axes = plt.subplots(1, 2, sharey=True, figsize=(12, 4.4),
|
||||
layout="constrained")
|
||||
figure_legend(fig, results, field, absolute=True, raptor_only=raptor_only)
|
||||
|
||||
for ax, arch in zip(axes, ARCHITECTURES):
|
||||
selected = [(row, compiler) for row, compiler in columns if row.arch == arch]
|
||||
positions = []
|
||||
for index, (row, _) in enumerate(selected):
|
||||
if index == 0:
|
||||
positions.append(0.0)
|
||||
continue
|
||||
previous = selected[index - 1][0]
|
||||
gap = 0.8 if row.model != previous.model else 0.4 if row.mode != previous.mode else 0.0
|
||||
positions.append(positions[-1] + 1.0 + gap)
|
||||
ax.set_xlabel("ARCHITECTURE A" if arch == "arch-a" else "ARCHITECTURE B",
|
||||
positions = grouped_positions([row for row, _ in selected])
|
||||
ax.set_xlabel(ARCH_NAMES[arch],
|
||||
fontsize=11, weight="bold", labelpad=8)
|
||||
ax.set_axisbelow(True)
|
||||
ax.set_ylim(0, y_max)
|
||||
@@ -384,14 +427,15 @@ def draw_absolute(results: list[Result], field: str, ylabel: str,
|
||||
for (_, mode), group in groupby(enumerate(selected), key=lambda item: (item[1][0].model, item[1][0].mode)):
|
||||
group_positions = [positions[position] for position, _ in group]
|
||||
centers.append((group_positions[0] + group_positions[-1]) / 2)
|
||||
labels.append(mode.capitalize())
|
||||
labels.append("Latency" if mode == "latency" else MODE_NAMES[mode])
|
||||
ax.set_xticks(centers, labels, minor=True)
|
||||
ax.tick_params(axis="x", which="minor", length=0, pad=4, labelsize=8)
|
||||
ax.set_ylabel(ylabel, fontsize=10.5)
|
||||
if arch == ARCHITECTURES[0]:
|
||||
ax.set_ylabel(ylabel, fontsize=10.5)
|
||||
if field == "throughput":
|
||||
ax.ticklabel_format(axis="y", style="plain", useOffset=False)
|
||||
ax.grid(axis="y", color="#E6EAF0", linewidth=0.8)
|
||||
ax.tick_params(axis="y", length=0, pad=6)
|
||||
ax.grid(axis="y")
|
||||
ax.tick_params(axis="y", labelleft=arch == ARCHITECTURES[0], length=0, pad=6)
|
||||
ax.tick_params(axis="x", which="major", length=0,
|
||||
pad=22 if {row.mode for row, _ in selected} == {"latency", "throughput"} else 6)
|
||||
|
||||
@@ -399,19 +443,22 @@ def draw_absolute(results: list[Result], field: str, ylabel: str,
|
||||
measurement = getattr(row, f"{compiler}_{field}") * scale
|
||||
if not math.isfinite(measurement):
|
||||
ax.text(x, y_max * 0.02, "N/A", rotation=90, ha="center",
|
||||
va="bottom", fontsize=8, color="#85909F")
|
||||
va="bottom", fontsize=8, color=TEXT_COLOR)
|
||||
continue
|
||||
status = getattr(row, f"{compiler}_validation")
|
||||
pattern = STAGE_PATTERNS[row.pipeline] if compiler == "raptor" else ""
|
||||
fill, border = COMPILER_STYLES[compiler]
|
||||
ax.bar(x, measurement, width=1.0,
|
||||
color=COMPILER_COLORS[compiler],
|
||||
color=fill,
|
||||
hatch=pattern,
|
||||
edgecolor=TEXT_COLOR if pattern else "none",
|
||||
linewidth=0.7 if pattern else 0.0, zorder=2)
|
||||
edgecolor=border, linewidth=0.7, zorder=2)
|
||||
label = (f"{measurement:,.2f}".rstrip("0").rstrip(".")
|
||||
if field == "throughput" else f"{measurement:.3g}")
|
||||
value_label(ax, x, measurement + y_max * 0.012, label, status)
|
||||
value_label(ax, x, measurement + y_max * 0.012, label, status, invalid_label="P. INVALID")
|
||||
|
||||
fit_bar_widths(fig)
|
||||
figure_legend(fig, results, field, absolute=True, raptor_only=raptor_only)
|
||||
fit_x_tick_labels(fig)
|
||||
fit_value_labels(fig)
|
||||
save_figure(fig, basename, output_dir)
|
||||
|
||||
@@ -419,43 +466,50 @@ def draw_absolute(results: list[Result], field: str, ylabel: str,
|
||||
def draw_pipeline_effect(results: list[Result], output_dir: Path) -> None:
|
||||
rows = [row for row in results if row.mode == "throughput" or row.pipeline == 1]
|
||||
stages = sorted({row.pipeline for row in rows})
|
||||
panels = [(model, color) for model, color in zip(MODELS, COLORS)
|
||||
if any(row.model == model for row in rows)]
|
||||
panels = [model for model in MODELS if any(row.model == model for row in rows)]
|
||||
lookup = {(row.arch, row.model, row.pipeline): row for row in rows}
|
||||
rates = {key: quotient(1000, row.raptor_latency) if row.mode == "latency" else row.raptor_throughput
|
||||
for key, row in lookup.items()}
|
||||
fig, axes = plt.subplots(len(ARCHITECTURES), len(panels), sharex=True, sharey="col",
|
||||
figsize=(3 * len(panels), 7.2), squeeze=False, layout="constrained")
|
||||
for column, (model, color) in enumerate(panels):
|
||||
figsize=(2.6 * len(panels), 4.3), squeeze=False, layout="constrained")
|
||||
fig.supylabel("Throughput (inferences/s)", fontsize=10.5)
|
||||
for column, model in enumerate(panels):
|
||||
fill, border = NETWORK_STYLES[model]
|
||||
for index, arch in enumerate(ARCHITECTURES):
|
||||
ax = axes[index, column]
|
||||
keys = [(arch, model, stage) for stage in stages]
|
||||
values = [rates.get(key, math.nan) for key in keys]
|
||||
valid = [key in lookup and lookup[key].raptor_validation == "PASS" for key in keys]
|
||||
ax.plot(stages, values, color=color, linewidth=1.8)
|
||||
ax.plot(stages, values, color=border, linewidth=1.8)
|
||||
for stage, value, passed in zip(stages, values, valid):
|
||||
if math.isfinite(value):
|
||||
marker = ("s" if stage == 1 else "o") if passed else "x"
|
||||
ax.plot(stage, value, color=color, marker=marker,
|
||||
status = lookup[(arch, model, stage)].raptor_validation
|
||||
ax.plot(stage, value, color=ERROR_COLOR if status == "FAIL" else border,
|
||||
markerfacecolor=fill, marker=marker,
|
||||
markersize=6, linestyle="none")
|
||||
ax.set_xticks(stages)
|
||||
ax.set_xlabel(model.upper(), fontsize=10.5, weight="bold", labelpad=8)
|
||||
ax.grid(axis="y", color="#E6EAF0", linewidth=0.8)
|
||||
ax.tick_params(axis="both", length=0, pad=6, labelbottom=True, labelleft=True)
|
||||
if index == 0:
|
||||
ax.set_title(MODEL_NAMES[model], fontsize=10.5, weight="normal", pad=8)
|
||||
ax.grid(axis="y")
|
||||
ax.tick_params(axis="both", length=0, pad=6,
|
||||
labelbottom=index == len(ARCHITECTURES) - 1, labelleft=True)
|
||||
ax.ticklabel_format(axis="y", style="plain", useOffset=False)
|
||||
if column == 0:
|
||||
ax.set_ylabel(f"{arch.replace('arch-', 'ARCHITECTURE ').upper()}\n"
|
||||
"Throughput (inferences/s)", fontsize=10.5)
|
||||
if column == len(panels) - 1:
|
||||
ax.yaxis.set_label_position("right")
|
||||
ax.set_ylabel(ARCH_NAMES[arch], fontsize=10.5, weight="bold", rotation=270, labelpad=24)
|
||||
for ax in axes.flat:
|
||||
ax.set_ylim(bottom=0)
|
||||
handles = [Line2D([], [], color="#657081", marker=marker, linestyle="none", label=label)
|
||||
for marker, label in (("s", "Derived as 1000ms / latency_ms"),
|
||||
ax.margins(y=0.08)
|
||||
handles = [Line2D([], [], color=TEXT_COLOR, markerfacecolor="white",
|
||||
marker=marker, linestyle="none", label=label)
|
||||
for marker, label in (("s", "Derived as 1 / latency"),
|
||||
("o", "Throughput measurement"))]
|
||||
if any(row.raptor_validation != "PASS" and math.isfinite(rates[key]) for key, row in lookup.items()):
|
||||
handles.append(Line2D([], [], color="#424B58", marker="x", linestyle="none",
|
||||
handles.append(Line2D([], [], color=TEXT_COLOR, marker="x", linestyle="none",
|
||||
label="Raptor validation not PASS"))
|
||||
fig.legend(handles=handles, loc="outside upper center", ncol=len(handles), frameon=False, fontsize=9.5)
|
||||
add_legend(fig, handles)
|
||||
fig.supxlabel("Raptor pipeline stages", fontsize=11)
|
||||
fit_x_tick_labels(fig)
|
||||
save_figure(fig, "raptor_pipeline_throughput", output_dir)
|
||||
|
||||
|
||||
@@ -471,6 +525,7 @@ def main() -> None:
|
||||
print(f"Results: {len(results)} configurations, {valid} valid comparisons, "
|
||||
f"{len(results)-valid} comparisons without two PASS statuses")
|
||||
draw_validation(results, args.output_dir)
|
||||
draw_validation(results, args.output_dir, compact=True)
|
||||
draw_pipeline_effect(results, args.output_dir)
|
||||
yolo_only_models = {
|
||||
model for model in MODELS if model.startswith("yolo")
|
||||
@@ -480,12 +535,19 @@ def main() -> None:
|
||||
}
|
||||
comparisons = [row for row in results if row.model not in yolo_only_models]
|
||||
yolo_only = [row for row in results if row.model in yolo_only_models]
|
||||
compact_speedups = select_compact_speedups(comparisons)
|
||||
for metric, label in (("speedup", "Speedup (x)"),
|
||||
("energy", "Energy-per-inference advantage (x)"),
|
||||
("power", "Average power advantage (x)")):
|
||||
for suffix, diverging in (("comparison", False), ("diverging", True)):
|
||||
draw_metric(compact_speedups, metric, label,
|
||||
f"{metric}_compact_{suffix}", args.output_dir, diverging=diverging, compact=True)
|
||||
for metric, name, label, mode in [
|
||||
("latency_speedup", "longest_stage_latency", "Longest stage latency speedup (x)", None),
|
||||
("latency_speedup", "latency", "Single-inference latency speedup (x)", "latency"),
|
||||
("throughput_speedup", "throughput", "Throughput speedup (x)", "throughput"),
|
||||
("power", "power", "Average power advantage (x)", None),
|
||||
("energy", "energy", "Energy per inference advantage (x)", None),
|
||||
("energy", "energy", "Energy-per-inference advantage (x)", None),
|
||||
]:
|
||||
selected = [row for row in comparisons if mode is None or row.mode == mode]
|
||||
draw_metric(selected, metric, label,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
"""Shared appearance and PNG/SVG export for PIM result plots."""
|
||||
"""Shared appearance and PNG/SVG/PDF export for PIM result plots."""
|
||||
|
||||
from pathlib import Path
|
||||
from math import ceil
|
||||
@@ -12,9 +12,32 @@ from matplotlib.patheffects import AbstractPathEffect
|
||||
from matplotlib.transforms import Affine2D, IdentityTransform, TransformedPath
|
||||
|
||||
|
||||
TEXT_COLOR = "#243449"
|
||||
HATCH_COLOR = "#C62828"
|
||||
COLORS = ("#2B65A1", "#168C8B", "#D18D32", "#8062A5", "#B65D60")
|
||||
PALETTE = {
|
||||
"sage": ("#B3D5CA", "#4B7C6A"),
|
||||
"sand": ("#E1BC8B", "#856A44"),
|
||||
"pale_blue": ("#EDF3FA", "#456B90"),
|
||||
"blue": ("#B3CCE3", "#456B90"),
|
||||
"pale_green": ("#EAF5F1", "#3E8271"),
|
||||
"pale_sand": ("#FCF1E5", "#A87940"),
|
||||
"red": ("#FF9494", "#C52020"),
|
||||
"purple": ("#D1C0E0", "#77558F"),
|
||||
}
|
||||
TEXT_COLOR = "#222222"
|
||||
MATRIX_TEXT_COLOR = "#1D2530"
|
||||
ERROR_COLOR = PALETTE["red"][1]
|
||||
GRID_COLOR = "#E6E9EC"
|
||||
LINE_COLOR = "#C9CED3"
|
||||
BAND_COLOR = "#F7F9FA"
|
||||
COMPILER_STYLES = {"raptor": PALETTE["sage"], "pimcomp": PALETTE["sand"]}
|
||||
METRIC_STYLES = {"latency": PALETTE["blue"], "throughput": PALETTE["sage"]}
|
||||
NETWORK_STYLES = {"vgg8": PALETTE["blue"], "resnet18": PALETTE["sage"],
|
||||
"resnet34": PALETTE["sand"], "googlenet": PALETTE["purple"],
|
||||
"yolo11n": PALETTE["red"]}
|
||||
COMPILER_NAMES = {"raptor": "Raptor", "pimcomp": "PIMCOMP"}
|
||||
ARCH_NAMES = {"arch-a": "Arch-A", "arch-b": "Arch-B"}
|
||||
MODEL_NAMES = {"vgg8": "VGG-8", "resnet18": "ResNet-18", "resnet34": "ResNet-34",
|
||||
"googlenet": "GoogLeNet", "yolo11n": "YOLO11n"}
|
||||
MODE_NAMES = {"latency": "Single-inference", "throughput": "Throughput"}
|
||||
|
||||
|
||||
class CenteredHatch(AbstractPathEffect):
|
||||
@@ -77,51 +100,130 @@ def configure_style() -> None:
|
||||
"text.color": TEXT_COLOR,
|
||||
"xtick.color": TEXT_COLOR,
|
||||
"ytick.color": TEXT_COLOR,
|
||||
"axes.edgecolor": "#CFD6DF",
|
||||
"axes.edgecolor": LINE_COLOR,
|
||||
"grid.color": GRID_COLOR,
|
||||
"grid.linewidth": 0.8,
|
||||
"hatch.color": TEXT_COLOR,
|
||||
"savefig.facecolor": "white",
|
||||
"figure.facecolor": "white",
|
||||
})
|
||||
|
||||
|
||||
def add_legend(fig, handles) -> None:
|
||||
"""Use the fewest balanced legend rows that fit the figure width."""
|
||||
renderer = fig.canvas.get_renderer()
|
||||
for rows in range(1, len(handles) + 1):
|
||||
columns = ceil(len(handles) / rows)
|
||||
grid = []
|
||||
start = 0
|
||||
for row in range(rows):
|
||||
count = len(handles) // rows + (row < len(handles) % rows)
|
||||
grid.append(handles[start:start + count] + [None] * (columns - count))
|
||||
start += count
|
||||
ordered = [handle for column in zip(*grid) for handle in column]
|
||||
legend = fig.legend(handles=[handle if handle is not None else Patch(visible=False) for handle in ordered],
|
||||
labels=[handle.get_label() if handle is not None else "" for handle in ordered],
|
||||
loc="outside upper center", ncols=columns, frameon=False, fontsize=9.5,
|
||||
handlelength=1.7, columnspacing=1.5, handletextpad=0.6, labelspacing=0.4)
|
||||
for handle, patch, text in zip(ordered, legend.legend_handles, legend.get_texts()):
|
||||
if handle is None:
|
||||
patch.set_visible(False)
|
||||
text.set_visible(False)
|
||||
if legend.get_window_extent(renderer).width <= fig.bbox.width - 20:
|
||||
return
|
||||
legend.remove()
|
||||
|
||||
|
||||
def value_label(ax, x, y, label: str, status: str = "PASS", direction: int = 1,
|
||||
fontsize: float = 8) -> None:
|
||||
fontsize: float = 8, invalid_label: str = "INVALID") -> None:
|
||||
alignment = "bottom" if direction > 0 else "top"
|
||||
rotation = 90 if direction > 0 else -90
|
||||
text = ax.text(x, y, label, rotation=rotation, ha="center", va=alignment,
|
||||
fontsize=fontsize, color=TEXT_COLOR)
|
||||
offset = direction * fontsize * 0.65 if status != "PASS" else 0
|
||||
ax.annotate(label, (x, y), xytext=(-offset, 0), textcoords="offset points",
|
||||
rotation=rotation, ha="center", va=alignment, fontsize=fontsize,
|
||||
color=TEXT_COLOR)
|
||||
if status != "PASS":
|
||||
height = text.get_window_extent(ax.figure.canvas.get_renderer()).height
|
||||
offset = direction * (height * 72 / ax.figure.dpi + 3)
|
||||
ax.annotate("INVALID" if status == "FAIL" else status, (x, y),
|
||||
xytext=(0, offset), textcoords="offset points",
|
||||
ax.annotate(invalid_label if status == "FAIL" else status, (x, y),
|
||||
xytext=(offset, 0), textcoords="offset points",
|
||||
rotation=rotation, ha="center", va=alignment, fontsize=fontsize,
|
||||
color=HATCH_COLOR)
|
||||
color=ERROR_COLOR if status == "FAIL" else TEXT_COLOR)
|
||||
|
||||
|
||||
def fit_x_tick_labels(fig) -> None:
|
||||
"""Rotate only overlapping labels, clockwise by 30 degrees."""
|
||||
fig.canvas.draw()
|
||||
renderer = fig.canvas.get_renderer()
|
||||
for minor in (False, True):
|
||||
labels = [(text, text.get_window_extent(renderer))
|
||||
for ax in fig.axes for text in ax.get_xticklabels(minor=minor)
|
||||
if text.get_visible() and text.get_text()]
|
||||
crowded = set()
|
||||
for index, (text, bounds) in enumerate(labels):
|
||||
for other_text, other_bounds in labels[:index]:
|
||||
if bounds.overlaps(other_bounds):
|
||||
crowded.update((text, other_text))
|
||||
for text in crowded:
|
||||
text.set_rotation(-30)
|
||||
text.set_rotation_mode("anchor")
|
||||
text.set_ha("right" if text.get_va() == "bottom" else "left")
|
||||
fig.canvas.draw()
|
||||
renderer = fig.canvas.get_renderer()
|
||||
for ax in fig.axes:
|
||||
bottom_minor = [text for text in ax.get_xticklabels(minor=True)
|
||||
if text.get_va() == "top" and text.get_visible() and text.get_text()]
|
||||
if bottom_minor:
|
||||
height = max(text.get_window_extent(renderer).height for text in bottom_minor)
|
||||
ax.tick_params(axis="x", which="major", pad=height * 72 / fig.dpi + 6)
|
||||
fig.canvas.draw()
|
||||
|
||||
|
||||
def fit_bar_widths(fig) -> None:
|
||||
"""Reserve enough horizontal space for each bar's rendered annotations."""
|
||||
fig.canvas.draw()
|
||||
renderer = fig.canvas.get_renderer()
|
||||
scale = 1.0
|
||||
for ax in fig.axes:
|
||||
labels = {}
|
||||
for text in ax.texts:
|
||||
if hasattr(text, "xy"):
|
||||
labels.setdefault(text.xy[0], []).append(text.get_window_extent(renderer))
|
||||
for container in ax.containers:
|
||||
for bar in container.patches:
|
||||
bounds = labels.get(bar.get_x() + bar.get_width() / 2, [])
|
||||
if bounds:
|
||||
width = max(box.x1 for box in bounds) - min(box.x0 for box in bounds)
|
||||
scale = max(scale, (width + renderer.points_to_pixels(2)) / bar.get_window_extent(renderer).width)
|
||||
if scale > 1:
|
||||
fig.set_size_inches(fig.get_figwidth() * scale, fig.get_figheight())
|
||||
|
||||
|
||||
def fit_value_labels(fig) -> None:
|
||||
"""Reserve vertical space for rotated labels using their rendered size."""
|
||||
"""Fit common vertical limits to the data and rendered value labels."""
|
||||
for ax in fig.axes:
|
||||
for text in ax.texts:
|
||||
text.set_in_layout(False)
|
||||
fig.canvas.draw()
|
||||
renderer = fig.canvas.get_renderer()
|
||||
limits = []
|
||||
labels = []
|
||||
for ax in fig.axes:
|
||||
bounds = ax.get_window_extent(renderer)
|
||||
labels = []
|
||||
for text in ax.texts:
|
||||
x, y = text.xy if hasattr(text, "xy") else text.get_position()
|
||||
anchor = ax.transData.transform((x, y))[1]
|
||||
extent = text.get_window_extent(renderer)
|
||||
labels.append((y, max(0, anchor - extent.y0 + 3) / bounds.height,
|
||||
max(0, extent.y1 - anchor + 3) / bounds.height))
|
||||
lower, upper = ax.get_ylim()
|
||||
for _ in range(12):
|
||||
lower = min([lower, *((y - below * upper) / (1 - below) for y, below, _ in labels)])
|
||||
upper = max([upper, *((y - above * lower) / (1 - above) for y, _, above in labels)])
|
||||
limits.append((lower, upper))
|
||||
limits.append(ax.dataLim.intervaly if ax.has_data() else ax.get_ylim())
|
||||
lower = min(0, *(lower for lower, _ in limits))
|
||||
upper = max(0, *(upper for _, upper in limits))
|
||||
if lower == upper:
|
||||
upper = lower + 1
|
||||
for _ in range(12):
|
||||
lower = min([lower, *((y - below * upper) / (1 - below) for y, below, _ in labels)])
|
||||
upper = max([upper, *((y - above * lower) / (1 - above) for y, _, above in labels)])
|
||||
for ax in fig.axes:
|
||||
ax.set_ylim(min(lower for lower, _ in limits), max(upper for _, upper in limits))
|
||||
ax.set_ylim(lower, upper)
|
||||
|
||||
|
||||
def save_figure(fig, basename: str, output_dir: Path) -> None:
|
||||
@@ -129,7 +231,7 @@ def save_figure(fig, basename: str, output_dir: Path) -> None:
|
||||
hatch = patch.get_hatch() or ""
|
||||
if any(symbol in hatch for symbol in "./\\"):
|
||||
patch.set_path_effects([CenteredHatch()])
|
||||
for extension in ("png", "svg"):
|
||||
for extension in ("png", "svg", "pdf"):
|
||||
target = output_dir / extension / f"{basename}.{extension}"
|
||||
target.parent.mkdir(parents=True, exist_ok=True)
|
||||
fig.savefig(target, dpi=220, bbox_inches="tight", pad_inches=0.04)
|
||||
|
||||
Reference in New Issue
Block a user