diff --git a/validation/operations/validation_results.csv b/validation/operations/validation_results.csv new file mode 100644 index 0000000..7aac0ab --- /dev/null +++ b/validation/operations/validation_results.csv @@ -0,0 +1,178 @@ +Operation,Arch,Result (l),Result (t),Compile (l),Host mem (l),Cores mem (l),Cores (l),Xbars (l),Latency (l),Power (l),Energy (l),Compile (t),Host mem (t),Cores mem (t),Cores (t),Xbars (t),Avg latency (t),Throughput (t),Avg power (t),Avg energy (t) +add/after_gemm,arch-a,PASS,PASS,0.040 s,0.01 MiB,0.01 MiB,4,4,SKIP,SKIP,SKIP,0.041 s,0.01 MiB,0.01 MiB,5,4,SKIP,SKIP,SKIP,SKIP +add/basic,arch-a,PASS,PASS,0.033 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.034 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP +add/broadcast_row,arch-a,PASS,PASS,0.034 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.034 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP +add/channel_broadcast_1024,arch-a,PASS,PASS,0.035 s,0.02 MiB,0.01 MiB,1,0,SKIP,SKIP,SKIP,0.035 s,0.02 MiB,0.01 MiB,1,0,SKIP,SKIP,SKIP,SKIP +add/leading_dimension_broadcast,arch-a,PASS,PASS,0.035 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.035 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP +concat/channel_axis,arch-a,PASS,PASS,0.035 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.035 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP +concat/negative_axis,arch-a,PASS,PASS,0.033 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.035 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP +concat/three_inputs_channel_axis,arch-a,PASS,PASS,0.036 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.035 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP +conv/batch_2,arch-a,PASS,PASS,0.042 s,0.00 MiB,0.00 MiB,2,2,SKIP,SKIP,SKIP,0.050 s,0.00 MiB,0.01 MiB,4,2,SKIP,SKIP,SKIP,SKIP +conv/batch_4_pointwise,arch-a,PASS,PASS,0.044 s,0.00 MiB,0.01 MiB,4,4,SKIP,SKIP,SKIP,0.052 s,0.01 MiB,0.01 MiB,5,4,SKIP,SKIP,SKIP,SKIP +conv/depthwise_1024_channels,arch-a,PASS,PASS,0.068 s,0.19 MiB,0.38 MiB,128,128,SKIP,SKIP,SKIP,0.116 s,0.26 MiB,0.50 MiB,130,128,SKIP,SKIP,SKIP,SKIP +conv/depthwise_grouped,arch-a,PASS,PASS,0.039 s,0.01 MiB,0.00 MiB,4,4,SKIP,SKIP,SKIP,0.044 s,0.01 MiB,0.00 MiB,6,4,SKIP,SKIP,SKIP,SKIP +conv/dilated_3x3,arch-a,PASS,PASS,0.050 s,0.01 MiB,0.01 MiB,9,9,SKIP,SKIP,SKIP,0.050 s,0.01 MiB,0.01 MiB,10,9,SKIP,SKIP,SKIP,SKIP +conv/dynamic,arch-a,PASS,PASS,0.039 s,0.00 MiB,0.00 MiB,5,0,SKIP,SKIP,SKIP,0.045 s,0.00 MiB,0.00 MiB,5,0,SKIP,SKIP,SKIP,SKIP +conv/explicit_padding,arch-a,PASS,PASS,0.042 s,0.01 MiB,0.02 MiB,16,16,SKIP,SKIP,SKIP,0.055 s,0.01 MiB,0.02 MiB,17,16,SKIP,SKIP,SKIP,SKIP +conv/grouped_many_groups,arch-a,PASS,PASS,0.547 s,0.05 MiB,0.09 MiB,64,64,SKIP,SKIP,SKIP,0.628 s,0.05 MiB,0.14 MiB,124,64,SKIP,SKIP,SKIP,SKIP +conv/grouped_two_groups,arch-a,PASS,PASS,0.049 s,0.00 MiB,0.00 MiB,2,2,SKIP,SKIP,SKIP,0.053 s,0.00 MiB,0.00 MiB,4,2,SKIP,SKIP,SKIP,SKIP +conv/huge_pointwise_1024,arch-a,PASS,PASS,0.160 s,0.01 MiB,0.09 MiB,64,64,SKIP,SKIP,SKIP,0.190 s,0.02 MiB,0.11 MiB,74,64,SKIP,SKIP,SKIP,SKIP +conv/huge_pointwise_1024_dynamic,arch-a,PASS,PASS,0.073 s,8.04 MiB,12.55 MiB,167,0,SKIP,SKIP,SKIP,0.249 s,8.05 MiB,12.45 MiB,165,0,SKIP,SKIP,SKIP,SKIP +conv/input_224_7x7_stride2,arch-a,PASS,PASS,0.777 s,24.14 MiB,61.87 MiB,168,169,SKIP,SKIP,SKIP,3.669 s,31.18 MiB,68.93 MiB,165,169,SKIP,SKIP,SKIP,SKIP +conv/kernel_2x2,arch-a,PASS,PASS,0.041 s,0.00 MiB,0.00 MiB,1,1,SKIP,SKIP,SKIP,0.063 s,0.00 MiB,0.00 MiB,3,1,SKIP,SKIP,SKIP,SKIP +conv/kernel_3x3,arch-a,PASS,PASS,0.043 s,0.01 MiB,0.01 MiB,9,9,SKIP,SKIP,SKIP,0.046 s,0.01 MiB,0.01 MiB,11,9,SKIP,SKIP,SKIP,SKIP +conv/kernel_equals_input_spatial,arch-a,PASS,PASS,0.042 s,0.00 MiB,0.00 MiB,2,2,SKIP,SKIP,SKIP,0.048 s,0.00 MiB,0.00 MiB,4,2,SKIP,SKIP,SKIP,SKIP +conv/large_input_channels_1x1,arch-a,PASS,PASS,0.079 s,0.01 MiB,0.01 MiB,8,8,SKIP,SKIP,SKIP,0.081 s,0.01 MiB,0.02 MiB,10,8,SKIP,SKIP,SKIP,SKIP +conv/large_output_channels_1x1,arch-a,PASS,PASS,0.081 s,0.00 MiB,0.01 MiB,8,8,SKIP,SKIP,SKIP,0.087 s,0.01 MiB,0.02 MiB,9,8,SKIP,SKIP,SKIP,SKIP +conv/large_spatial,arch-a,PASS,PASS,0.048 s,0.01 MiB,0.04 MiB,36,36,SKIP,SKIP,SKIP,0.069 s,0.02 MiB,0.04 MiB,37,36,SKIP,SKIP,SKIP,SKIP +conv/multi_channel,arch-a,PASS,PASS,0.043 s,0.00 MiB,0.00 MiB,3,3,SKIP,SKIP,SKIP,0.046 s,0.00 MiB,0.00 MiB,4,3,SKIP,SKIP,SKIP,SKIP +conv/non_square_kernel_1x3,arch-a,PASS,PASS,0.045 s,0.00 MiB,0.00 MiB,2,2,SKIP,SKIP,SKIP,0.047 s,0.00 MiB,0.00 MiB,3,2,SKIP,SKIP,SKIP,SKIP +conv/non_square_kernel_3x1,arch-a,PASS,PASS,0.042 s,0.00 MiB,0.00 MiB,2,2,SKIP,SKIP,SKIP,0.046 s,0.00 MiB,0.00 MiB,3,2,SKIP,SKIP,SKIP,SKIP +conv/non_uniform_stride,arch-a,PASS,PASS,0.045 s,0.00 MiB,0.00 MiB,3,3,SKIP,SKIP,SKIP,0.046 s,0.00 MiB,0.00 MiB,4,3,SKIP,SKIP,SKIP,SKIP +conv/output_channel_grouping_minimal,arch-a,PASS,PASS,0.065 s,0.10 MiB,0.28 MiB,128,128,SKIP,SKIP,SKIP,0.104 s,0.10 MiB,0.25 MiB,128,128,SKIP,SKIP,SKIP,SKIP +conv/pointwise_1x1,arch-a,PASS,PASS,0.044 s,0.00 MiB,0.00 MiB,1,1,SKIP,SKIP,SKIP,0.044 s,0.00 MiB,0.00 MiB,2,1,SKIP,SKIP,SKIP,SKIP +conv/pointwise_tiled_chain,arch-a,PASS,PASS,0.738 s,0.01 MiB,0.03 MiB,17,80,SKIP,SKIP,SKIP,0.745 s,0.02 MiB,0.08 MiB,22,80,SKIP,SKIP,SKIP,SKIP +conv/real_asymmetric_padding,arch-a,PASS,PASS,0.044 s,0.01 MiB,0.03 MiB,28,28,SKIP,SKIP,SKIP,0.060 s,0.01 MiB,0.03 MiB,29,28,SKIP,SKIP,SKIP,SKIP +conv/relu_conv_store,arch-a,PASS,PASS,0.070 s,0.16 MiB,0.66 MiB,168,168,SKIP,SKIP,SKIP,0.120 s,0.17 MiB,0.66 MiB,166,166,SKIP,SKIP,SKIP,SKIP +conv/same_lower_3x3,arch-a,PASS,PASS,0.043 s,0.01 MiB,0.02 MiB,25,25,SKIP,SKIP,SKIP,0.053 s,0.01 MiB,0.02 MiB,26,25,SKIP,SKIP,SKIP,SKIP +conv/same_padding_3x3,arch-a,PASS,PASS,0.046 s,0.01 MiB,0.02 MiB,25,25,SKIP,SKIP,SKIP,0.056 s,0.01 MiB,0.02 MiB,26,25,SKIP,SKIP,SKIP,SKIP +conv/strategy_depthwise_16,arch-a,PASS,PASS,0.062 s,0.06 MiB,0.33 MiB,168,168,SKIP,SKIP,SKIP,0.125 s,0.08 MiB,0.11 MiB,27,52,SKIP,SKIP,SKIP,SKIP +conv/strategy_input_k_tiled,arch-a,PASS,PASS,0.061 s,0.08 MiB,0.27 MiB,108,108,SKIP,SKIP,SKIP,0.091 s,0.08 MiB,0.32 MiB,110,108,SKIP,SKIP,SKIP,SKIP +conv/strategy_output_channel_tiled,arch-a,PASS,PASS,0.062 s,0.03 MiB,0.15 MiB,72,72,SKIP,SKIP,SKIP,0.078 s,0.04 MiB,0.13 MiB,73,72,SKIP,SKIP,SKIP,SKIP +conv/strategy_streamed_packed,arch-a,PASS,PASS,0.131 s,3.34 MiB,7.89 MiB,168,168,SKIP,SKIP,SKIP,0.418 s,3.35 MiB,7.89 MiB,165,165,SKIP,SKIP,SKIP,SKIP +conv/strategy_streamed_patch,arch-a,PASS,PASS,0.074 s,0.34 MiB,1.32 MiB,168,168,SKIP,SKIP,SKIP,0.159 s,0.35 MiB,1.32 MiB,165,165,SKIP,SKIP,SKIP,SKIP +conv/strategy_tiled_2d,arch-a,PASS,PASS,0.096 s,0.10 MiB,0.37 MiB,168,168,SKIP,SKIP,SKIP,0.179 s,0.11 MiB,0.54 MiB,165,168,SKIP,SKIP,SKIP,SKIP +conv/stride_2,arch-a,PASS,PASS,0.053 s,0.01 MiB,0.00 MiB,4,4,SKIP,SKIP,SKIP,0.053 s,0.01 MiB,0.00 MiB,6,4,SKIP,SKIP,SKIP,SKIP +conv/with_bias_3x3,arch-a,PASS,PASS,0.045 s,0.00 MiB,0.01 MiB,3,3,SKIP,SKIP,SKIP,0.047 s,0.00 MiB,0.01 MiB,4,3,SKIP,SKIP,SKIP,SKIP +conv/with_constant,arch-a,PASS,PASS,0.057 s,0.00 MiB,0.00 MiB,1,1,SKIP,SKIP,SKIP,0.069 s,0.00 MiB,0.00 MiB,3,1,SKIP,SKIP,SKIP,SKIP +conv/without_kernel_shape_attr,arch-a,PASS,PASS,0.051 s,0.01 MiB,0.01 MiB,9,9,SKIP,SKIP,SKIP,0.057 s,0.01 MiB,0.01 MiB,11,9,SKIP,SKIP,SKIP,SKIP +conv/yolo11n_depthwise_head,arch-a,PASS,PASS,2.129 s,8.66 MiB,33.48 MiB,164,250,SKIP,SKIP,SKIP,9.599 s,10.25 MiB,36.90 MiB,166,255,SKIP,SKIP,SKIP,SKIP +conv/yolo11n_heavy,arch-a,PASS,PASS,0.188 s,4.82 MiB,19.00 MiB,160,800,SKIP,SKIP,SKIP,0.298 s,4.83 MiB,19.10 MiB,161,800,SKIP,SKIP,SKIP,SKIP +conv/yolo11n_stem,arch-a,PASS,PASS,0.873 s,12.85 MiB,31.85 MiB,168,488,SKIP,SKIP,SKIP,1.318 s,12.88 MiB,31.68 MiB,165,485,SKIP,SKIP,SKIP,SKIP +div/after_gemm,arch-a,PASS,PASS,0.046 s,0.01 MiB,0.01 MiB,4,4,SKIP,SKIP,SKIP,0.052 s,0.01 MiB,0.01 MiB,5,4,SKIP,SKIP,SKIP,SKIP +div/basic,arch-a,PASS,PASS,0.042 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.044 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP +div/channel_broadcast_1024,arch-a,PASS,PASS,0.048 s,0.02 MiB,0.01 MiB,1,0,SKIP,SKIP,SKIP,0.040 s,0.02 MiB,0.01 MiB,1,0,SKIP,SKIP,SKIP,SKIP +div/leading_dimension_broadcast,arch-a,PASS,PASS,0.053 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.058 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP +div/runtime_scalar_rhs,arch-a,PASS,PASS,0.043 s,0.02 MiB,0.01 MiB,1,0,SKIP,SKIP,SKIP,0.050 s,0.02 MiB,0.01 MiB,1,0,SKIP,SKIP,SKIP,SKIP +div/scalar_constant,arch-a,PASS,PASS,0.047 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.043 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP +gather/3d_input_axis1,arch-a,PASS,PASS,0.041 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.045 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP +gather/axis0_matrix_indices,arch-a,PASS,PASS,0.040 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.039 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP +gather/axis1,arch-a,PASS,PASS,0.040 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.040 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP +gather/negative_axis,arch-a,PASS,PASS,0.041 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.042 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP +gather/negative_indices,arch-a,PASS,PASS,0.061 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.049 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP +gemm/alpha_beta,arch-a,PASS,PASS,0.050 s,0.01 MiB,0.01 MiB,4,4,SKIP,SKIP,SKIP,0.048 s,0.01 MiB,0.01 MiB,5,4,SKIP,SKIP,SKIP,SKIP +gemm/bias_rank2_broadcast,arch-a,PASS,PASS,0.045 s,0.00 MiB,0.01 MiB,4,4,SKIP,SKIP,SKIP,0.062 s,0.01 MiB,0.01 MiB,5,4,SKIP,SKIP,SKIP,SKIP +gemm/dynamic,arch-a,PASS,PASS,0.043 s,0.00 MiB,0.00 MiB,4,0,SKIP,SKIP,SKIP,0.049 s,0.00 MiB,0.00 MiB,5,0,SKIP,SKIP,SKIP,SKIP +gemm/dynamic_alpha,arch-a,PASS,PASS,0.045 s,0.00 MiB,0.00 MiB,4,0,SKIP,SKIP,SKIP,0.045 s,0.00 MiB,0.00 MiB,5,0,SKIP,SKIP,SKIP,SKIP +gemm/dynamic_beta,arch-a,PASS,PASS,0.057 s,0.00 MiB,0.00 MiB,4,0,SKIP,SKIP,SKIP,0.058 s,0.00 MiB,0.00 MiB,5,0,SKIP,SKIP,SKIP,SKIP +gemm/dynamic_bias,arch-a,PASS,PASS,0.047 s,0.00 MiB,0.00 MiB,4,0,SKIP,SKIP,SKIP,0.053 s,0.00 MiB,0.00 MiB,5,0,SKIP,SKIP,SKIP,SKIP +gemm/dynamic_bias_alpha_beta,arch-a,PASS,PASS,0.043 s,0.00 MiB,0.00 MiB,4,0,SKIP,SKIP,SKIP,0.046 s,0.00 MiB,0.00 MiB,5,0,SKIP,SKIP,SKIP,SKIP +gemm/dynamic_transpose_b,arch-a,PASS,PASS,0.076 s,0.00 MiB,0.00 MiB,4,0,SKIP,SKIP,SKIP,0.077 s,0.00 MiB,0.00 MiB,5,0,SKIP,SKIP,SKIP,SKIP +gemm/huge_1024,arch-a,PASS,PASS,0.186 s,0.01 MiB,0.09 MiB,64,64,SKIP,SKIP,SKIP,0.190 s,0.02 MiB,0.10 MiB,73,64,SKIP,SKIP,SKIP,SKIP +gemm/large,arch-a,PASS,PASS,0.050 s,0.02 MiB,0.03 MiB,16,16,SKIP,SKIP,SKIP,0.048 s,0.02 MiB,0.03 MiB,17,16,SKIP,SKIP,SKIP,SKIP +gemm/large_k_small_n,arch-a,PASS,PASS,0.113 s,0.01 MiB,0.01 MiB,8,8,SKIP,SKIP,SKIP,0.088 s,0.01 MiB,0.01 MiB,9,8,SKIP,SKIP,SKIP,SKIP +gemm/non_square,arch-a,PASS,PASS,0.046 s,0.00 MiB,0.01 MiB,4,4,SKIP,SKIP,SKIP,0.049 s,0.01 MiB,0.01 MiB,5,4,SKIP,SKIP,SKIP,SKIP +gemm/scalar_bias,arch-a,PASS,PASS,0.050 s,0.00 MiB,0.01 MiB,4,4,SKIP,SKIP,SKIP,0.057 s,0.01 MiB,0.01 MiB,5,4,SKIP,SKIP,SKIP,SKIP +gemm/small,arch-a,PASS,PASS,0.047 s,0.00 MiB,0.00 MiB,2,2,SKIP,SKIP,SKIP,0.079 s,0.00 MiB,0.00 MiB,4,2,SKIP,SKIP,SKIP,SKIP +gemm/small_k_large_n,arch-a,PASS,PASS,0.113 s,0.00 MiB,0.01 MiB,8,8,SKIP,SKIP,SKIP,0.110 s,0.01 MiB,0.02 MiB,9,8,SKIP,SKIP,SKIP,SKIP +gemm/square_weights,arch-a,PASS,PASS,0.073 s,0.03 MiB,0.08 MiB,40,40,SKIP,SKIP,SKIP,0.084 s,0.03 MiB,0.09 MiB,43,40,SKIP,SKIP,SKIP,SKIP +gemm/transpose_a,arch-a,PASS,PASS,0.055 s,0.00 MiB,0.01 MiB,4,4,SKIP,SKIP,SKIP,0.052 s,0.00 MiB,0.01 MiB,6,4,SKIP,SKIP,SKIP,SKIP +gemm/transpose_a_and_b,arch-a,PASS,PASS,0.048 s,0.00 MiB,0.01 MiB,4,4,SKIP,SKIP,SKIP,0.053 s,0.00 MiB,0.01 MiB,6,4,SKIP,SKIP,SKIP,SKIP +gemm/transpose_b,arch-a,PASS,PASS,0.065 s,0.00 MiB,0.01 MiB,4,4,SKIP,SKIP,SKIP,0.051 s,0.01 MiB,0.01 MiB,5,4,SKIP,SKIP,SKIP,SKIP +gemm/transpose_b_with_bias,arch-a,PASS,PASS,0.053 s,0.01 MiB,0.01 MiB,4,4,SKIP,SKIP,SKIP,0.060 s,0.01 MiB,0.01 MiB,5,4,SKIP,SKIP,SKIP,SKIP +gemm/with_bias,arch-a,PASS,PASS,0.054 s,0.01 MiB,0.01 MiB,4,4,SKIP,SKIP,SKIP,0.044 s,0.01 MiB,0.01 MiB,5,4,SKIP,SKIP,SKIP,SKIP +gemv/all_constant,arch-a,PASS,PASS,0.047 s,0.00 MiB,0.00 MiB,0,0,SKIP,SKIP,SKIP,0.040 s,0.00 MiB,0.00 MiB,0,0,SKIP,SKIP,SKIP,SKIP +gemv/constant_weight,arch-a,PASS,PASS,0.058 s,0.00 MiB,0.01 MiB,5,4,SKIP,SKIP,SKIP,0.059 s,0.00 MiB,0.01 MiB,7,4,SKIP,SKIP,SKIP,SKIP +gemv/non_uniform_bias,arch-a,PASS,PASS,0.062 s,0.00 MiB,0.01 MiB,5,4,SKIP,SKIP,SKIP,0.061 s,0.00 MiB,0.01 MiB,7,4,SKIP,SKIP,SKIP,SKIP +gemv/scalar_bias,arch-a,PASS,PASS,0.061 s,0.00 MiB,0.01 MiB,5,4,SKIP,SKIP,SKIP,0.072 s,0.00 MiB,0.01 MiB,7,4,SKIP,SKIP,SKIP,SKIP +gemv/uniform_bias,arch-a,PASS,PASS,0.076 s,0.00 MiB,0.01 MiB,5,4,SKIP,SKIP,SKIP,0.064 s,0.00 MiB,0.01 MiB,7,4,SKIP,SKIP,SKIP,SKIP +matmul/basic,arch-a,PASS,PASS,0.071 s,0.00 MiB,0.00 MiB,2,2,SKIP,SKIP,SKIP,0.072 s,0.00 MiB,0.00 MiB,4,2,SKIP,SKIP,SKIP,SKIP +matmul/batched_3d,arch-a,PASS,PASS,0.066 s,0.00 MiB,0.01 MiB,4,4,SKIP,SKIP,SKIP,0.060 s,0.00 MiB,0.01 MiB,5,4,SKIP,SKIP,SKIP,SKIP +matmul/batched_3d_dynamic,arch-a,PASS,PASS,0.048 s,0.00 MiB,0.00 MiB,4,0,SKIP,SKIP,SKIP,0.042 s,0.00 MiB,0.00 MiB,4,0,SKIP,SKIP,SKIP,SKIP +matmul/batched_left_constant,arch-a,PASS,PASS,0.048 s,0.00 MiB,0.02 MiB,8,8,SKIP,SKIP,SKIP,0.054 s,0.01 MiB,0.02 MiB,9,8,SKIP,SKIP,SKIP,SKIP +matmul/batched_lhs_broadcast,arch-a,PASS,PASS,0.047 s,0.00 MiB,0.01 MiB,4,4,SKIP,SKIP,SKIP,0.056 s,0.00 MiB,0.01 MiB,5,4,SKIP,SKIP,SKIP,SKIP +matmul/batched_rhs_broadcast,arch-a,PASS,PASS,0.052 s,0.00 MiB,0.01 MiB,4,4,SKIP,SKIP,SKIP,0.053 s,0.00 MiB,0.01 MiB,5,4,SKIP,SKIP,SKIP,SKIP +matmul/dynamic,arch-a,PASS,PASS,0.044 s,0.00 MiB,0.00 MiB,4,0,SKIP,SKIP,SKIP,0.054 s,0.00 MiB,0.00 MiB,5,0,SKIP,SKIP,SKIP,SKIP +matmul/huge_1024,arch-a,PASS,PASS,0.174 s,0.01 MiB,0.09 MiB,64,64,SKIP,SKIP,SKIP,0.188 s,0.02 MiB,0.10 MiB,73,64,SKIP,SKIP,SKIP,SKIP +matmul/left_constant,arch-a,PASS,PASS,0.051 s,0.00 MiB,0.01 MiB,4,4,SKIP,SKIP,SKIP,0.058 s,0.00 MiB,0.01 MiB,6,4,SKIP,SKIP,SKIP,SKIP +matmul/matrix_vector,arch-a,PASS,PASS,0.120 s,0.52 MiB,0.77 MiB,165,170,SKIP,SKIP,SKIP,0.239 s,0.53 MiB,0.77 MiB,165,173,SKIP,SKIP,SKIP,SKIP +matmul/vector_matrix,arch-a,PASS,PASS,0.088 s,0.01 MiB,0.01 MiB,8,8,SKIP,SKIP,SKIP,0.087 s,0.01 MiB,0.01 MiB,9,8,SKIP,SKIP,SKIP,SKIP +matmul/yolo_attention,arch-a,PASS,PASS,0.394 s,1.02 MiB,39.22 MiB,168,0,SKIP,SKIP,SKIP,0.512 s,2.65 MiB,26.70 MiB,43,0,SKIP,SKIP,SKIP,SKIP +mul/after_conv,arch-a,PASS,PASS,0.065 s,0.00 MiB,0.00 MiB,3,3,SKIP,SKIP,SKIP,0.081 s,0.00 MiB,0.00 MiB,4,3,SKIP,SKIP,SKIP,SKIP +mul/after_conv_scalar_constant,arch-a,PASS,PASS,0.053 s,0.00 MiB,0.00 MiB,3,3,SKIP,SKIP,SKIP,0.055 s,0.00 MiB,0.00 MiB,4,3,SKIP,SKIP,SKIP,SKIP +mul/basic,arch-a,PASS,PASS,0.075 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.079 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP +mul/channel_broadcast_1024,arch-a,PASS,PASS,0.038 s,0.02 MiB,0.01 MiB,1,0,SKIP,SKIP,SKIP,0.039 s,0.02 MiB,0.01 MiB,1,0,SKIP,SKIP,SKIP,SKIP +mul/leading_dimension_broadcast,arch-a,PASS,PASS,0.038 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.037 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP +mul/scalar_constant,arch-a,PASS,PASS,0.038 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.036 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP +pool/avg_basic,arch-a,PASS,PASS,0.047 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.045 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP +pool/avg_ceil_mode,arch-a,PASS,PASS,0.061 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.056 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP +pool/avg_explicit_padding,arch-a,PASS,PASS,0.047 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.051 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP +pool/avg_include_pad,arch-a,PASS,PASS,0.049 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.050 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP +pool/avg_large_channels,arch-a,PASS,PASS,0.056 s,0.04 MiB,0.02 MiB,1,0,SKIP,SKIP,SKIP,0.059 s,0.04 MiB,0.02 MiB,1,0,SKIP,SKIP,SKIP,SKIP +pool/avg_non_uniform_stride,arch-a,PASS,PASS,0.050 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.053 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP +pool/avg_real_asymmetric_padding,arch-a,PASS,PASS,0.048 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.048 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP +pool/max_after_conv,arch-a,PASS,PASS,0.052 s,0.00 MiB,0.00 MiB,4,4,SKIP,SKIP,SKIP,0.054 s,0.00 MiB,0.00 MiB,5,4,SKIP,SKIP,SKIP,SKIP +pool/max_basic,arch-a,PASS,PASS,0.040 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.039 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP +pool/max_ceil_mode,arch-a,PASS,PASS,0.042 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.044 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP +pool/max_global_style_kernel_equals_input,arch-a,PASS,PASS,0.050 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.041 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP +pool/max_non_square_kernel,arch-a,PASS,PASS,0.044 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.067 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP +pool/max_real_asymmetric_padding,arch-a,PASS,PASS,0.059 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.059 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP +pool/max_same_upper,arch-a,PASS,PASS,0.059 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.049 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP +pool/max_stride2_multichannel,arch-a,PASS,PASS,0.045 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.062 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP +reduce_mean/4d_spatial,arch-a,PASS,PASS,0.044 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.046 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP +reduce_mean/4d_spatial_keepdims_0,arch-a,PASS,PASS,0.044 s,0.00 MiB,0.00 MiB,3,0,SKIP,SKIP,SKIP,0.045 s,0.00 MiB,0.00 MiB,4,0,SKIP,SKIP,SKIP,SKIP +reduce_mean/after_conv,arch-a,PASS,PASS,0.069 s,0.00 MiB,0.00 MiB,3,3,SKIP,SKIP,SKIP,0.067 s,0.00 MiB,0.00 MiB,4,3,SKIP,SKIP,SKIP,SKIP +reduce_mean/all_axes_keepdims_0,arch-a,PASS,PASS,0.044 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.048 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP +reduce_mean/all_axes_keepdims_1,arch-a,PASS,PASS,0.041 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.052 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP +reduce_mean/basic,arch-a,PASS,PASS,0.048 s,0.00 MiB,0.00 MiB,4,0,SKIP,SKIP,SKIP,0.042 s,0.00 MiB,0.00 MiB,4,0,SKIP,SKIP,SKIP,SKIP +reduce_mean/channel_axis_nchw,arch-a,PASS,PASS,0.053 s,0.03 MiB,0.02 MiB,4,0,SKIP,SKIP,SKIP,0.048 s,0.03 MiB,0.02 MiB,4,0,SKIP,SKIP,SKIP,SKIP +reduce_mean/keepdims_0,arch-a,PASS,PASS,0.046 s,0.00 MiB,0.00 MiB,4,0,SKIP,SKIP,SKIP,0.041 s,0.00 MiB,0.00 MiB,5,0,SKIP,SKIP,SKIP,SKIP +reduce_mean/large_dimension_1024,arch-a,PASS,PASS,0.037 s,0.01 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.039 s,0.01 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP +reduce_mean/legacy_axes_1_2_keepdims_1,arch-a,PASS,PASS,0.041 s,0.00 MiB,0.00 MiB,2,0,SKIP,SKIP,SKIP,0.051 s,0.00 MiB,0.00 MiB,2,0,SKIP,SKIP,SKIP,SKIP +reduce_mean/legacy_axis1_keepdims_0,arch-a,PASS,PASS,0.049 s,0.00 MiB,0.00 MiB,8,0,SKIP,SKIP,SKIP,0.043 s,0.00 MiB,0.00 MiB,9,0,SKIP,SKIP,SKIP,SKIP +reduce_mean/legacy_axis1_keepdims_1,arch-a,PASS,PASS,0.043 s,0.00 MiB,0.00 MiB,8,0,SKIP,SKIP,SKIP,0.041 s,0.00 MiB,0.00 MiB,8,0,SKIP,SKIP,SKIP,SKIP +reduce_mean/legacy_empty_axes_noop,arch-a,PASS,PASS,0.035 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.040 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP +reduce_mean/legacy_nchw_spatial,arch-a,PASS,PASS,0.048 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.055 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP +reduce_mean/legacy_negative_axis,arch-a,PASS,PASS,0.050 s,0.00 MiB,0.00 MiB,6,0,SKIP,SKIP,SKIP,0.064 s,0.00 MiB,0.00 MiB,6,0,SKIP,SKIP,SKIP,SKIP +reduce_mean/legacy_reduce_all_keepdims_1,arch-a,PASS,PASS,0.040 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.041 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP +reduce_mean/negative_axis,arch-a,PASS,PASS,0.044 s,0.00 MiB,0.00 MiB,6,0,SKIP,SKIP,SKIP,0.046 s,0.00 MiB,0.00 MiB,6,0,SKIP,SKIP,SKIP,SKIP +relu/4d,arch-a,PASS,PASS,0.043 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.039 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP +relu/after_conv,arch-a,PASS,PASS,0.061 s,0.00 MiB,0.00 MiB,3,3,SKIP,SKIP,SKIP,0.062 s,0.00 MiB,0.00 MiB,4,3,SKIP,SKIP,SKIP,SKIP +relu/after_gemm,arch-a,PASS,PASS,0.044 s,0.01 MiB,0.01 MiB,4,4,SKIP,SKIP,SKIP,0.045 s,0.01 MiB,0.01 MiB,5,4,SKIP,SKIP,SKIP,SKIP +relu/basic,arch-a,PASS,PASS,0.039 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.038 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP +reshape/4d_to_2d_flatten,arch-a,PASS,PASS,0.042 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.045 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP +reshape/infer_dim_minus_one,arch-a,PASS,PASS,0.041 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.040 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP +reshape/same_rank,arch-a,PASS,PASS,0.040 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.039 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP +reshape/zero_copies_input_dim,arch-a,PASS,PASS,0.042 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.036 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP +resize/height_only,arch-a,PASS,PASS,0.047 s,0.00 MiB,0.00 MiB,4,0,SKIP,SKIP,SKIP,0.047 s,0.00 MiB,0.00 MiB,4,0,SKIP,SKIP,SKIP,SKIP +resize/nearest_2x,arch-a,PASS,PASS,0.042 s,0.00 MiB,0.00 MiB,4,0,SKIP,SKIP,SKIP,0.051 s,0.00 MiB,0.00 MiB,4,0,SKIP,SKIP,SKIP,SKIP +resize/nearest_downsample,arch-a,PASS,PASS,0.042 s,0.00 MiB,0.00 MiB,2,0,SKIP,SKIP,SKIP,0.045 s,0.00 MiB,0.00 MiB,2,0,SKIP,SKIP,SKIP,SKIP +resize/non_uniform_scales,arch-a,PASS,PASS,0.042 s,0.00 MiB,0.00 MiB,6,0,SKIP,SKIP,SKIP,0.046 s,0.00 MiB,0.00 MiB,6,0,SKIP,SKIP,SKIP,SKIP +resize/width_only,arch-a,PASS,PASS,0.043 s,0.00 MiB,0.00 MiB,2,0,SKIP,SKIP,SKIP,0.043 s,0.00 MiB,0.00 MiB,2,0,SKIP,SKIP,SKIP,SKIP +resize/with_sizes,arch-a,PASS,PASS,0.045 s,0.00 MiB,0.00 MiB,3,0,SKIP,SKIP,SKIP,0.047 s,0.00 MiB,0.00 MiB,3,0,SKIP,SKIP,SKIP,SKIP +sigmoid/4d,arch-a,PASS,PASS,0.040 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.058 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP +sigmoid/after_gemm,arch-a,PASS,PASS,0.048 s,0.01 MiB,0.01 MiB,4,4,SKIP,SKIP,SKIP,0.051 s,0.01 MiB,0.01 MiB,5,4,SKIP,SKIP,SKIP,SKIP +sigmoid/basic,arch-a,PASS,PASS,0.038 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.053 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP +slice/2d_basic,arch-a,PASS,PASS,0.039 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.043 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP +slice/after_conv,arch-a,PASS,PASS,0.050 s,0.00 MiB,0.01 MiB,6,6,SKIP,SKIP,SKIP,0.059 s,0.00 MiB,0.01 MiB,7,6,SKIP,SKIP,SKIP,SKIP +slice/default_axes,arch-a,PASS,PASS,0.039 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.038 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP +slice/large_channel_1024,arch-a,PASS,PASS,0.041 s,0.01 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.039 s,0.01 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP +slice/nchw_spatial_crop,arch-a,PASS,PASS,0.044 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.044 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP +slice/negative_axis,arch-a,PASS,PASS,0.055 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.042 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP +slice/negative_indices,arch-a,PASS,PASS,0.043 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.038 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP +slice/step2,arch-a,PASS,PASS,0.041 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.042 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP +softmax/3d_last_axis,arch-a,PASS,PASS,0.044 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.042 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP +softmax/basic,arch-a,PASS,PASS,0.040 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.038 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP +softmax/channel_axis,arch-a,PASS,PASS,0.037 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.047 s,0.00 MiB,0.00 MiB,3,0,SKIP,SKIP,SKIP,SKIP +softmax/large_dimension_1024,arch-a,PASS,PASS,0.036 s,0.01 MiB,0.01 MiB,1,0,SKIP,SKIP,SKIP,0.038 s,0.01 MiB,0.01 MiB,1,0,SKIP,SKIP,SKIP,SKIP +softmax/negative_axis,arch-a,PASS,PASS,0.069 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.060 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP +split/basic,arch-a,PASS,PASS,0.052 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.057 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP +split/equal_three_way,arch-a,PASS,PASS,0.042 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.042 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP +split/negative_axis,arch-a,PASS,PASS,0.058 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.043 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP +split/uneven_channel_axis_4d,arch-a,PASS,PASS,0.048 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.040 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP +sub/after_gemm,arch-a,PASS,PASS,0.050 s,0.01 MiB,0.01 MiB,4,4,SKIP,SKIP,SKIP,0.047 s,0.01 MiB,0.01 MiB,5,4,SKIP,SKIP,SKIP,SKIP +sub/basic,arch-a,PASS,PASS,0.040 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.043 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP +sub/broadcast_row,arch-a,PASS,PASS,0.056 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.044 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP +sub/channel_broadcast_1024,arch-a,PASS,PASS,0.052 s,0.02 MiB,0.01 MiB,1,0,SKIP,SKIP,SKIP,0.043 s,0.02 MiB,0.01 MiB,1,0,SKIP,SKIP,SKIP,SKIP +sub/constant_lhs_broadcast,arch-a,PASS,PASS,0.044 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.038 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP +sub/leading_dimension_broadcast,arch-a,PASS,PASS,0.040 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,0.033 s,0.00 MiB,0.00 MiB,1,0,SKIP,SKIP,SKIP,SKIP diff --git a/validation/tests/test_pim_experiments.py b/validation/tests/test_pim_experiments.py index e93089b..a227f8b 100644 --- a/validation/tests/test_pim_experiments.py +++ b/validation/tests/test_pim_experiments.py @@ -3,7 +3,6 @@ from __future__ import annotations import argparse import copy import csv -import importlib.util import json import os from pathlib import Path @@ -48,26 +47,6 @@ def test_ablation_results_stay_in_independent_output_roots(tmp_path): assert rows[1]["latency"] == change -@pytest.mark.parametrize("custom_input", [False, True]) -def test_ablation_plots_follow_the_selected_experiment(tmp_path, monkeypatch, custom_input): - spec = importlib.util.spec_from_file_location("ablation_plots", TOOLS / "ablation/generate_plots.py") - plots = importlib.util.module_from_spec(spec) - monkeypatch.setitem(sys.modules, spec.name, plots) - spec.loader.exec_module(plots) - expected = tmp_path / "experiment/results_ablation.csv" if custom_input else ( - ablation.DEFAULT_RESULTS_PATH - ) - monkeypatch.setattr(sys, "argv", ["generate_plots.py", *(["--input", str(expected)] if custom_input else [])]) - result = plots.Result("baseline", "arch-a", "vgg8", "latency", 1, "PASS", {}) - inputs, outputs = [], [] - monkeypatch.setattr(plots, "configure_style", lambda: None) - monkeypatch.setattr(plots, "load_results", lambda path: inputs.append(path) or [result]) - monkeypatch.setattr(plots, "draw_metric", lambda rows, metric, label, output: outputs.append(output)) - plots.main() - assert inputs == [expected] - assert outputs == [expected.parent / "plots"] * 3 - - @pytest.fixture def recorded_failure(tmp_path): model, functional, config = (tmp_path / name for name in ("model.onnx", "functional.onnx", "config.json")) diff --git a/validation/tests/test_pim_plots.py b/validation/tests/test_pim_plots.py index cecb551..954700d 100644 --- a/validation/tests/test_pim_plots.py +++ b/validation/tests/test_pim_plots.py @@ -1,6 +1,7 @@ from __future__ import annotations import csv +from dataclasses import replace import importlib.util import math from pathlib import Path @@ -14,16 +15,14 @@ TOOLS = Path(__file__).resolve().parents[1] / "tools/pim" @pytest.fixture -def plotters(monkeypatch): - modules = [] - for name, directory in (("comparison_plots", "pimcomp/compare"), ("ablation_plots", "ablation")): - monkeypatch.syspath_prepend(str(TOOLS / directory)) - spec = importlib.util.spec_from_file_location(name, TOOLS / directory / "generate_plots.py") - module = importlib.util.module_from_spec(spec) - monkeypatch.setitem(sys.modules, name, module) - spec.loader.exec_module(module) - modules.append(module) - return modules +def comparison(monkeypatch): + directory = TOOLS / "pimcomp/compare" + monkeypatch.syspath_prepend(str(directory)) + spec = importlib.util.spec_from_file_location("comparison_plots", directory / "generate_plots.py") + module = importlib.util.module_from_spec(spec) + monkeypatch.setitem(sys.modules, spec.name, module) + spec.loader.exec_module(module) + return module @pytest.fixture @@ -42,8 +41,7 @@ def write_csv(path, fields, rows): @pytest.mark.parametrize("custom", [False, True]) -def test_comparison_plot_cli_input_paths(plotters, tmp_path, monkeypatch, custom): - comparison, _ = plotters +def test_comparison_plot_cli_input_paths(comparison, tmp_path, monkeypatch, custom): expected = tmp_path / "custom.csv" if custom else TOOLS / "pimcomp/compare/results_comparison.csv" seen = [] @@ -62,8 +60,7 @@ def test_comparison_plot_cli_input_paths(plotters, tmp_path, monkeypatch, custom assert seen == [expected] -def test_exact_comparison_configuration_matrix(plotters, comparison_csv, tmp_path): - comparison, _ = plotters +def test_exact_comparison_configuration_matrix(comparison, comparison_csv, tmp_path): fields, rows = comparison_csv results = comparison.load_results(write_csv(tmp_path / "results.csv", fields, rows)) keys = {(row.arch, row.model, row.mode, row.pipeline) for row in results} @@ -74,8 +71,7 @@ def test_exact_comparison_configuration_matrix(plotters, comparison_csv, tmp_pat @pytest.mark.parametrize("damage,message", [("missing", "Missing"), ("duplicate", "Duplicate"), ("arch", "Unexpected"), ("model", "Unexpected"), ("pipeline", "Unexpected"), ("mode", "Unexpected")]) -def test_comparison_rejects_bad_matrix_keys(plotters, comparison_csv, tmp_path, damage, message): - comparison, _ = plotters +def test_comparison_rejects_bad_matrix_keys(comparison, comparison_csv, tmp_path, damage, message): fields, rows = comparison_csv if damage == "missing": rows.pop() @@ -108,50 +104,431 @@ else: @pytest.mark.parametrize("field,value", [("pimcomp_latency_ms", "999"), ("pimcomp_throughput_samples_s", "999"), ("pimcomp_power_mw", "999"), ("pimcomp_energy_pj", "999"), ("pimcomp_functional_validation", "FAIL")]) -def test_repeated_pimcomp_batch_results_must_agree(plotters, comparison_csv, tmp_path, field, value): - comparison, _ = plotters +def test_repeated_pimcomp_batch_results_must_agree(comparison, comparison_csv, tmp_path, field, value): fields, rows = comparison_csv row = next(row for row in rows if row["arch"] == "arch-a" and row["model"] == "vgg8" and row["raptor_pipeline"] == "4") row[field] = value - with pytest.raises(ValueError, match="inconsistent shared Pimcomp batch results"): + with pytest.raises(ValueError, match="inconsistent shared PIMCOMP batch results"): comparison.load_results(write_csv(tmp_path / "results.csv", fields, rows)) -def test_legacy_invalid_ablation_pairs_are_unavailable_without_changing_pass(plotters): - _, ablation = plotters - with pytest.warns(UserWarning, match="2 invalid reciprocal"): - rows = ablation.load_results(TOOLS / "ablation/results_ablation.csv") - invalid = [row for row in rows if row.model == "googlenet" and row.variant == "no-plan" - and row.mode == "throughput"] - assert len(invalid) == 2 - assert all(row.validation == "PASS" and math.isnan(row.metrics["latency"]) - and math.isnan(row.metrics["throughput"]) for row in invalid) +@pytest.mark.parametrize("status,expected", [("PASS", ["1.0"]), + ("FAIL", ["1.0", "INVALID"]), ("SKIPPED", ["1.0", "SKIPPED"]), + ("NA", ["1.0", "NA"])]) +@pytest.mark.parametrize("direction", [1, -1]) +def test_value_label_validation_text(comparison, status, expected, direction): + fig, ax = comparison.plt.subplots() + try: + comparison.value_label(ax, 0, 1, "1.0", status, direction) + assert [text.get_text() for text in ax.texts] == expected + assert ax.texts[-1].get_color() == (comparison.ERROR_COLOR if status == "FAIL" else comparison.TEXT_COLOR) + if status != "PASS": + value, validation = ax.texts + assert value.xy == validation.xy + assert value.get_position()[1] == validation.get_position()[1] == 0 + assert value.get_position()[0] == -validation.get_position()[0] != 0 + finally: + comparison.plt.close(fig) -def test_ablation_missing_values_and_legitimate_extremes(plotters, tmp_path): - _, ablation = plotters - row = {"variant": "no-plan", "arch": "arch-a", "model": "vgg8", "mode": "throughput", - "raptor_pipeline": "4", "functional_validation": "FAIL", "latency": "-100.00%", - "throughput": "+100000000.00%", "power": "NA", "energy": "-100.00%"} - result = ablation.load_results(write_csv(tmp_path / "extreme.csv", list(row), [row]))[0] - assert result.validation == "FAIL" - assert result.metrics["latency"] == result.metrics["energy"] == -100 - assert result.metrics["throughput"] == 100000000 - assert math.isnan(result.metrics["power"]) +@pytest.mark.parametrize("values", [(1, 2), (-1, -2), (-1, 2), (0, 0)]) +def test_value_label_limits_are_tight_shared_and_unclipped(comparison, values): + fig, axes = comparison.plt.subplots(1, 2, sharey=True, figsize=(10, 4.4), layout="constrained") + try: + for ax, value in zip(axes, values): + ax.set_ylim(-10, 10) + ax.bar(0, value) + direction = -1 if value < 0 else 1 + comparison.value_label(ax, 0, value + direction * 0.05, f"{value:+.2f}%", "FAIL", direction) + comparison.fit_value_labels(fig) + fig.canvas.draw() + renderer = fig.canvas.get_renderer() + assert axes[0].get_ylim() == axes[1].get_ylim() + lower, upper = axes[0].get_ylim() + assert upper - lower < 20 + assert lower <= min(0, *values) and upper >= max(0, *values) + assert axes[0].get_window_extent(renderer).height == pytest.approx(axes[1].get_window_extent(renderer).height) + for ax in axes: + bounds = ax.get_window_extent(renderer) + for text in ax.texts: + extent = text.get_window_extent(renderer) + assert bounds.y0 <= extent.y0 <= extent.y1 <= bounds.y1 + finally: + comparison.plt.close(fig) -def test_both_plotters_export_png_and_svg(plotters, tmp_path): - comparison, ablation = plotters +@pytest.mark.parametrize("statuses,expected", [ + (("PASS", "PASS", "PASS"), "PASS"), + (("FAIL", "PASS", "PASS"), "FAIL"), + (("PASS", "FAIL", "PASS"), "FAIL"), + (("PASS", "PASS", "FAIL"), "FAIL"), + (("PASS", "SKIPPED", "NA"), "SKIPPED"), + (("PASS", "PASS", "NA"), "N/A"), +]) +def test_compact_validation_combines_pipeline_statuses(comparison, tmp_path, monkeypatch, statuses, expected): + rows = comparison.load_results(TOOLS / "pimcomp/compare/results_comparison.csv") + stages = dict(zip((2, 4, 8), statuses)) + rows = [replace(row, raptor_validation=stages[row.pipeline]) + if row.arch == "arch-a" and row.model == "vgg8" and row.mode == "throughput" + else row for row in rows] + captured = [] + monkeypatch.setattr(comparison, "save_figure", lambda fig, name, path: captured.append((fig, name))) + comparison.draw_validation(rows, tmp_path, compact=True) + fig, name = captured[0] + try: + assert name == "validation_status_compact" + assert fig.get_size_inches() == pytest.approx((8.5, 3.3)) + for index, ax in enumerate(fig.axes): + assert [label.get_text() for label in ax.get_xticklabels(minor=True)] == [ + "Raptor", "PIMCOMP", "Raptor", "PIMCOMP"] + assert [label.get_text() for label in ax.get_yticklabels()] == ([ + "VGG-8", "ResNet-18", "ResNet-34", "GoogLeNet", "YOLO11n"] if index == 0 else []) + assert len(ax.patches) == len(ax.texts) == 4 * len(comparison.MODELS) + assert [label.get_text() for label in ax.get_xticklabels()] == ["Latency", "Throughput"] + assert all(text.get_fontsize() == 12 and text.get_weight() == "normal" for text in ax.texts) + assert all(text.get_color() == "#1D2530" for text in ax.texts) + ax = fig.axes[0] + throughput_x = ax.xaxis.get_minorticklocs()[2] + assert [text.get_text() for text in ax.texts if text.get_position() == (throughput_x, 0)] == [expected] + finally: + comparison.plt.close(fig) + + +@pytest.mark.parametrize("rates,stage", [((300, 200, 100), 2), ((100, 300, 200), 4), + ((100, 200, 300), 8), ((math.nan, 200, 300), 8), ((300, 300, 200), 2), + ((math.nan, math.nan, math.nan), 2)]) +def test_compact_speedups_select_best_pipeline(comparison, rates, stage): + template = comparison.load_results(TOOLS / "pimcomp/compare/results_comparison.csv")[0] + rows = [replace(template, latency_speedup=0.5)] + rows += [replace(template, mode="throughput", pipeline=depth, raptor_throughput=rate, + pimcomp_throughput=100, raptor_validation="FAIL") + for depth, rate in zip((2, 4, 8), rates)] + latency, throughput = comparison.select_compact_speedups(rows) + assert latency is rows[0] and latency.speedup == 0.5 + assert throughput.pipeline == stage and throughput.raptor_validation == "FAIL" + + +@pytest.mark.parametrize("diverging", [False, True]) +@pytest.mark.parametrize("metric,ylabel", [("speedup", "Speedup (x)"), + ("energy", "Energy-per-inference advantage (x)"), ("power", "Average power advantage (x)")]) +def test_compact_metric_bars_and_units(comparison, tmp_path, monkeypatch, diverging, metric, ylabel): + from matplotlib.colors import to_rgba + + template = comparison.load_results(TOOLS / "pimcomp/compare/results_comparison.csv")[0] + rows = [replace(template, arch=arch, mode=mode, pipeline=stage, latency_speedup=0.5, + energy=0.5 if mode == "latency" else 3, power=0.5 if mode == "latency" else 3, + raptor_throughput=300, pimcomp_throughput=100, pimcomp_validation="FAIL") + for arch in comparison.ARCHITECTURES for mode, stage in (("latency", 1), ("throughput", 4))] + captured = [] + monkeypatch.setattr(comparison, "save_figure", lambda fig, name, path: captured.append(fig)) + comparison.draw_metric(rows, metric, ylabel, "compact", tmp_path, + diverging=diverging, compact=True) + fig = captured[0] + try: + assert fig.get_size_inches() == pytest.approx((8.5, 4.4)) + expected_legend = { + "speedup": ["Single-inference latency", "Best-case throughput"], + "energy": ["Single-inference energy", "Best-case throughput energy"], + "power": ["Single-inference power", "Best-case throughput power"], + } + assert [text.get_text() for text in fig.legends[0].get_texts()] == expected_legend[metric] + for index, ax in enumerate(fig.axes): + latency, throughput = (container.patches[0] for container in ax.containers) + lower, upper = ax.get_xlim() + assert latency.get_x() - lower == pytest.approx(0.4) + assert upper - throughput.get_x() - throughput.get_width() == pytest.approx(0.4) + assert latency.get_x() + latency.get_width() == throughput.get_x() + assert latency.get_facecolor() == to_rgba(comparison.METRIC_STYLES["latency"][0]) + assert throughput.get_facecolor() == to_rgba(comparison.METRIC_STYLES["throughput"][0]) + assert latency.get_edgecolor() == to_rgba(comparison.METRIC_STYLES["latency"][1]) + assert throughput.get_edgecolor() == to_rgba(comparison.METRIC_STYLES["throughput"][1]) + assert not latency.get_hatch() and throughput.get_hatch() == "//" + assert latency.get_height() == (-1 if diverging else 0.5) + assert throughput.get_height() == (2 if diverging else 3) + assert ax.get_ylabel() == (ylabel if index == 0 else "") + assert bool(ax.get_yticklabels()) == (index == 0) + assert all("×" not in label.get_text() for label in ax.get_yticklabels()) + assert [text.get_text() for text in ax.texts] == [ + "2.00×" if diverging else "0.50×", "P. INVALID", "3.00×", "P. INVALID"] + finally: + comparison.plt.close(fig) + + +@pytest.mark.parametrize("missing_energy", [False, True]) +def test_compact_metrics_use_speedup_pipeline_selection(comparison, monkeypatch, missing_energy): + rows = comparison.load_results(TOOLS / "pimcomp/compare/results_comparison.csv") + cases = {2: (100, 9, 8), 4: (300, math.nan if missing_energy else 0.5, 0.6), 8: (200, 7, 6)} + rows = [replace(row, raptor_throughput=cases[row.pipeline][0], + energy=cases[row.pipeline][1], power=cases[row.pipeline][2]) + if row.arch == "arch-a" and row.model == "vgg8" and row.mode == "throughput" + else row for row in rows] + captured = [] + + def capture(selected, field, ylabel, basename, output_dir, diverging=False, compact=False): + if compact: + captured.append((selected, field, basename, diverging)) + + monkeypatch.setattr(comparison, "load_results", lambda path: rows) + for name in ("configure_style", "draw_validation", "draw_pipeline_effect", "draw_absolute"): + monkeypatch.setattr(comparison, name, lambda *args, **kwargs: None) + monkeypatch.setattr(comparison, "draw_metric", capture) + monkeypatch.setattr(sys, "argv", ["generate_plots.py"]) + comparison.main() + assert {(field, basename, diverging) for _, field, basename, diverging in captured} == { + (metric, f"{metric}_compact_{suffix}", diverging) + for metric in ("speedup", "energy", "power") + for suffix, diverging in (("comparison", False), ("diverging", True))} + assert len(captured) == 6 + selected = captured[0][0] + assert all(plotted is selected for plotted, _, _, _ in captured) + throughput = next(row for row in selected if row.arch == "arch-a" and row.model == "vgg8" + and row.mode == "throughput") + assert throughput.pipeline == 4 and throughput.power == 0.6 + assert math.isnan(throughput.energy) if missing_energy else throughput.energy == 0.5 + + +def test_compiler_palette_and_grayscale_patterns(comparison, tmp_path, monkeypatch): + from matplotlib.colors import to_rgba + + assert comparison.COMPILER_STYLES == { + "raptor": ("#B3D5CA", "#4B7C6A"), "pimcomp": ("#E1BC8B", "#856A44")} + assert comparison.METRIC_STYLES == { + "latency": ("#B3CCE3", "#456B90"), "throughput": ("#B3D5CA", "#4B7C6A")} + assert comparison.VALIDATION_STYLES["PASS"] == ("#B3D5CA", "#4B7C6A") + assert comparison.TEXT_COLOR == "#222222" + assert comparison.NETWORK_STYLES["yolo11n"] == ("#FF9494", "#C52020") + assert comparison.NETWORK_STYLES["googlenet"] == ("#D1C0E0", "#77558F") + rows = comparison.load_results(TOOLS / "pimcomp/compare/results_comparison.csv") + selected = [replace(row, power=2 if row.arch == "arch-a" else 0.5) + for row in rows if row.model == "vgg8" and row.mode == "latency"] + captured = [] + monkeypatch.setattr(comparison, "save_figure", lambda fig, *args: captured.append(fig)) + comparison.draw_metric(selected, "power", "Average power advantage (x)", "power", tmp_path, diverging=True) + comparison.draw_absolute(selected, "latency", "Single-inference latency (ms)", "latency", tmp_path) + try: + for ax, compiler in zip(captured[0].axes, ("raptor", "pimcomp")): + bar = ax.containers[0].patches[0] + assert bar.get_facecolor() == to_rgba(comparison.COMPILER_STYLES[compiler][0]) + assert bar.get_edgecolor() == to_rgba(comparison.COMPILER_STYLES[compiler][1]) + assert not bar.get_hatch() + for ax in captured[1].axes: + assert [container.patches[0].get_hatch() for container in ax.containers] == ["", ""] + finally: + for fig in captured: + comparison.plt.close(fig) + + +def test_tick_rotation_changes_only_overlapping_labels(comparison): + fig, ax = comparison.plt.subplots(figsize=(4, 3), layout="constrained") + try: + ax.set_xlim(-1, 6) + ax.set_xticks([0, 0.5, 5], ["Single-inference", "Throughput", "8"]) + comparison.fit_x_tick_labels(fig) + assert [text.get_rotation() for text in ax.get_xticklabels()] == [330, 330, 0] + assert [text.get_ha() for text in ax.get_xticklabels()] == ["left", "left", "center"] + finally: + comparison.plt.close(fig) + + +def test_all_plot_terminology(comparison, monkeypatch): + from matplotlib.path import Path as PlotPath + from matplotlib.text import Text + from matplotlib.transforms import Affine2D + + plotter = comparison + labels = set() + + def capture(fig, name, output): + fig.canvas.draw() + if name != "raptor_pipeline_throughput": + assert fig.axes[1].get_ylabel() == "" + assert fig.axes[1].get_yticklabels() == [] + renderer = fig.canvas.get_renderer() + for ax in fig.axes: + for container in ax.containers: + for bar in container.patches: + center = bar.get_x() + bar.get_width() / 2 + bounds = bar.get_window_extent(renderer) + for text in ax.texts: + if hasattr(text, "xy") and text.xy[0] == center: + extent = text.get_window_extent(renderer) + assert bounds.x0 <= extent.x0 <= extent.x1 <= bounds.x1, (name, text.get_text()) + for legend in fig.legends: + bounds = legend.get_window_extent(renderer) + assert fig.bbox.x0 <= bounds.x0 <= bounds.x1 <= fig.bbox.x1, name + tick_paths = [] + for ax in fig.axes: + for minor in (False, True): + for text in ax.get_xticklabels(minor=minor): + if not text.get_visible() or not text.get_text(): + continue + angle = text.get_rotation() + assert angle in (0, 330), (name, text.get_text(), angle) + bounds = text.get_window_extent(renderer) + text.set_rotation(0) + unrotated = text.get_window_extent(renderer) + text.set_rotation(angle) + path = PlotPath.unit_rectangle().transformed( + Affine2D().scale(unrotated.width, unrotated.height).rotate_deg(angle)) + rotated = path.get_extents() + path = path.transformed(Affine2D().translate(bounds.x0 - rotated.x0, bounds.y0 - rotated.y0)) + assert not any(path.intersects_path(other, filled=True) for other in tick_paths), name + tick_paths.append(path) + if text.get_text() in comparison.MODEL_NAMES.values(): + assert angle == 0, (name, text.get_text()) + heights = [ax.get_window_extent(renderer).height for ax in fig.axes] + assert max(heights) - min(heights) < 1, name + for ax in fig.axes: + bounds = ax.get_window_extent(renderer) + for text in ax.texts: + extent = text.get_window_extent(renderer) + assert bounds.y0 - 1 <= extent.y0 <= extent.y1 <= bounds.y1 + 1, (name, text.get_text()) + assert fig.bbox.x0 <= extent.x0 <= extent.x1 <= fig.bbox.x1, (name, text.get_text()) + labels.update(text.get_text() for text in fig.findobj(Text)) + plotter.plt.close(fig) + + monkeypatch.setattr(plotter, "save_figure", capture) + monkeypatch.setattr(sys, "argv", ["generate_plots.py"]) + plotter.main() + text = "\n".join(labels) + for old in ("Pimcomp", "ARCHITECTURE", "VGG8", "RESNET", "GOOGLENET", "YOLO11N", + "Single inference", "Best case", "Latency, 1 stage", "Energy per inference advantage"): + assert old not in text + assert {"VGG-8", "ResNet-18", "ResNet-34", "GoogLeNet"} <= labels + assert "Arch-A" in text and "Arch-B" in text + assert {"Raptor", "PIMCOMP", "YOLO11n", "Single-inference", "2 stages", "4 stages", "8 stages", + "Single-inference latency (ms)", "Throughput (inferences/s)", "Energy per inference (mJ)", + "Energy-per-inference advantage (x)", "Derived as 1 / latency", + "Throughput measurement"} <= labels + + +@pytest.mark.parametrize("width,one_row", [(12, True), (4.5, False), (3, False)]) +def test_legend_uses_available_width_and_balances_rows(comparison, width, one_row): + from collections import Counter + from matplotlib.patches import Patch + + fig = comparison.plt.figure(figsize=(width, 4), layout="constrained") + try: + handles = [Patch(label=f"Entry {index}") for index in range(7)] + comparison.add_legend(fig, handles) + fig.canvas.draw() + renderer = fig.canvas.get_renderer() + legend = fig.legends[0] + counts = Counter(round(text.get_window_extent(renderer).y0, 3) + for text in legend.get_texts() if text.get_visible()) + assert sum(counts.values()) == len(handles) + assert (len(counts) == 1) == one_row + assert max(counts.values()) - min(counts.values()) <= 1 + bounds = legend.get_window_extent(renderer) + assert 0 <= bounds.x0 <= bounds.x1 <= fig.bbox.width + finally: + comparison.plt.close(fig) + + +@pytest.mark.parametrize("diverging", [False, True]) +def test_advantage_throughput_groups_touch(comparison, tmp_path, monkeypatch, diverging): + rows = comparison.load_results(TOOLS / "pimcomp/compare/results_comparison.csv") + selected = [row for row in rows if row.model in ("vgg8", "resnet18")] + captured = [] + monkeypatch.setattr(comparison, "save_figure", lambda fig, *args: captured.append(fig)) + comparison.draw_metric(selected, "energy", "Energy-per-inference advantage (x)", "energy", tmp_path, + diverging=diverging) + fig = captured[0] + try: + for ax in fig.axes: + bars = [container.patches[0] for container in ax.containers] + lower, upper = ax.get_xlim() + assert bars[0].get_x() - lower == pytest.approx(0.4) + assert upper - bars[-1].get_x() - bars[-1].get_width() == pytest.approx(0.4) + for index, (left, right) in enumerate(zip(bars, bars[1:])): + edge = left.get_x() + left.get_width() + if index % 4 in (1, 2): + assert edge == pytest.approx(right.get_x()) + else: + assert edge < right.get_x() + finally: + comparison.plt.close(fig) + + +def test_grouped_absolute_plot_scheduling_labels(comparison, tmp_path, monkeypatch): + rows = comparison.load_results(TOOLS / "pimcomp/compare/results_comparison.csv") + captured = [] + monkeypatch.setattr(comparison, "save_figure", lambda fig, *args: captured.append(fig)) + comparison.draw_absolute(rows, "energy", "Energy per inference (mJ)", "energy", tmp_path, scale=1e-9) + fig = captured[0] + try: + for ax in fig.axes: + assert {text.get_text() for text in ax.get_xticklabels(minor=True)} == {"Latency", "Throughput"} + labels = [text.get_text() for text in fig.legends[0].get_texts()] + assert "1 stage (latency)" in labels and "Single-inference" not in labels + finally: + comparison.plt.close(fig) + + +def test_pipeline_plot_connects_derived_and_measured_rates(comparison, tmp_path, monkeypatch): + rows = comparison.load_results(TOOLS / "pimcomp/compare/results_comparison.csv") + captured = [] + monkeypatch.setattr(comparison, "save_figure", lambda fig, *args: captured.append(fig)) + comparison.draw_pipeline_effect(rows, tmp_path) + fig = captured[0] + try: + assert fig.get_size_inches() == pytest.approx((13, 4.3)) + assert fig._supylabel.get_text() == "Throughput (inferences/s)" + for index, (ax, (arch, model)) in enumerate(zip(fig.axes, ((arch, model) for arch in comparison.ARCHITECTURES + for model in comparison.MODELS))): + selected = [row for row in rows if row.arch == arch and row.model == model] + measured_line, derived_marker, *measured_markers = ax.lines + assert measured_line.get_ydata()[0] == pytest.approx(1000 / selected[0].raptor_latency) + assert list(measured_line.get_ydata()[1:]) == [row.raptor_throughput for row in selected[1:]] + assert derived_marker.get_marker() == "s" + assert derived_marker.get_ydata()[0] == pytest.approx(1000 / selected[0].raptor_latency) + assert derived_marker.get_color() == comparison.NETWORK_STYLES[model][1] + assert all(marker.get_marker() == "o" for marker in measured_markers) + assert measured_line.get_color() == comparison.NETWORK_STYLES[model][1] + assert [label.get_text() for label in ax.get_xticklabels()] == ( + ["1", "2", "4", "8"] if arch == "arch-b" else []) + assert ax.get_xlabel() == "" + assert ax.get_title() == (comparison.MODEL_NAMES[model] if arch == "arch-a" else "") + assert ax.title.get_weight() == "normal" + if model == comparison.MODELS[-1]: + assert ax.get_ylabel() == comparison.ARCH_NAMES[arch] + assert ax.yaxis.get_label_position() == "right" + assert ax.yaxis.label.get_rotation() == 270 + assert ax.yaxis.label.get_weight() == "bold" + else: + assert ax.get_ylabel() == "" + assert all(label.get_rotation() == 0 for label in ax.get_xticklabels()) + lower, upper = ax.get_ylim() + values = measured_line.get_ydata() + assert 0 < lower < min(values) <= max(values) < upper + assert ax.get_yticklabels() + assert ax.get_ylim() == fig.axes[index % len(comparison.MODELS)].get_ylim() + assert [text.get_text() for text in fig.legends[0].get_texts()] == [ + "Derived as 1 / latency", "Throughput measurement"] + assert len({ax.lines[0].get_color() for ax in fig.axes[:len(comparison.MODELS)]}) == len(comparison.MODELS) + finally: + comparison.plt.close(fig) + + +def test_plotter_exports_png_svg_and_pdf(comparison, tmp_path): comparison.configure_style() rows = comparison.load_results(TOOLS / "pimcomp/compare/results_comparison.csv") - selected = [row for row in rows if row.model == "vgg8" and row.mode == "latency"] + selected = [replace(row, raptor_validation="FAIL", pimcomp_latency=math.nan) + for row in rows if row.model == "vgg8" and row.mode == "latency"] comparison.draw_absolute(selected, "latency", "Latency (ms)", "comparison-smoke", tmp_path) - row = ablation.Result("baseline", "arch-a", "vgg8", "latency", 1, "PASS", - {"latency": math.nan, "power": 0, "energy": 0, "throughput": math.nan}) - ablation.draw_metric([row], "latency", "Latency change (%)", tmp_path) - for name in ("comparison-smoke", "latency_pipeline1_latency"): + comparison.draw_validation(rows, tmp_path, compact=True) + compact_names = [] + for metric in ("speedup", "energy", "power"): + for suffix, diverging in (("comparison", False), ("diverging", True)): + name = f"{metric}_compact_{suffix}" + compact_names.append(name) + comparison.draw_metric(comparison.select_compact_speedups(rows), metric, "Advantage (x)", + name, tmp_path, diverging=diverging, compact=True) + for name in ("comparison-smoke", "validation_status_compact", *compact_names): assert (tmp_path / f"png/{name}.png").read_bytes().startswith(b"\x89PNG\r\n\x1a\n") svg = (tmp_path / f"svg/{name}.svg").read_text() assert " float: - if value == "NA": - return math.nan - if not value.endswith("%"): - raise ValueError(f"Expected a percentage or NA, got {value!r}") - parsed = float(value[:-1]) - if not math.isfinite(parsed): - raise ValueError(f"Non-finite percentage: {value!r}") - return parsed - - -def load_results(path: Path) -> list[Result]: - rows = [] - seen = set() - invalid_pairs = [] - with path.open(newline="", encoding="utf-8-sig") as stream: - reader = csv.DictReader(stream) - if not {"variant", *RESULT_FIELDS} <= set(reader.fieldnames or []): - raise ValueError("Expected the combined ablation CSV with variant and percentage columns; " - "a per-variant results_comparison.csv cannot be used here.") - for row in reader: - variant, mode, status = row["variant"], row["mode"], row["functional_validation"] - if variant not in VARIANT_COLORS or mode not in ("latency", "throughput") or status not in STATUSES: - raise ValueError(f"Unknown ablation variant, mode, or validation status: {variant}, {mode}, {status}") - pipeline = int(row["raptor_pipeline"]) - key = (variant, row["arch"], row["model"], mode, pipeline) - if key in seen: - raise ValueError(f"Duplicate ablation configuration: {key}") - seen.add(key) - metrics = {metric: percentage(row[metric]) - for metric in ("latency", "throughput", "power", "energy")} - # Each may round to -100% alone; reciprocal metrics cannot both do so. - if mode == "throughput" and metrics["latency"] == metrics["throughput"] == -100: - metrics["latency"] = metrics["throughput"] = math.nan - invalid_pairs.append(key) - rows.append(Result(variant, row["arch"], row["model"], mode, pipeline, status, metrics)) - if not rows: - raise ValueError("The ablation CSV contains no results") - if invalid_pairs: - warnings.warn(f"{len(invalid_pairs)} invalid reciprocal latency/throughput percentage pairs; " - f"displaying N/A. Examples: {invalid_pairs[:3]}", stacklevel=2) - return rows - - -def draw_metric(results: list[Result], metric: str, ylabel: str, output_dir: Path) -> None: - architectures = sorted({row.arch for row in results}) - present_models = {row.model for row in results} - models = [model for model in ABLATION_DEFAULT_MODELS if model in present_models] - models += sorted(present_models - set(models)) - variants = [name for name in VARIANT_COLORS if any(row.variant == name for row in results)] - lookup = {(row.arch, row.model, row.variant): row for row in results} - values = [row.metrics[metric] for row in results if math.isfinite(row.metrics[metric])] - lower, upper = min(0, min(values, default=0)), max(0, max(values, default=0)) - padding = max(upper - lower, 1) * 0.35 - fig, axes = plt.subplots(1, len(architectures), sharey=True, squeeze=False, - figsize=(7 * len(architectures), 5.2), layout="constrained") - handles = [Patch(facecolor=VARIANT_COLORS[name], label=name) for name in variants] - if len(values) < len(architectures) * len(models) * len(variants): - handles.append(Patch(facecolor="white", edgecolor="#B2BAC5", label="N/A: missing measurement")) - fig.legend(handles=handles, loc="outside upper center", ncol=4, frameon=False, fontsize=9.5) - width = 0.8 / len(variants) - mode, pipeline = results[0].mode, results[0].pipeline - for ax, arch in zip(axes[0], architectures): - ax.set_axisbelow(True) - ax.set_ylim(lower - padding if lower < 0 else 0, upper + padding) - ax.set_xlim(-0.5, len(models) - 0.5) - ax.set_xticks(range(len(models)), [model.upper() for model in models], fontsize=9.5) - ax.set_xlabel(f"{arch.replace('arch-', 'ARCHITECTURE ').upper()}\n" - f"{mode.capitalize()}, {pipeline} stage" + ("s" if pipeline != 1 else ""), - fontsize=11, weight="bold", labelpad=8) - ax.set_ylabel(ylabel, fontsize=10.5) - ax.yaxis.set_major_formatter(PercentFormatter(xmax=100, decimals=0)) - ax.tick_params(axis="both", length=0, pad=6, labelleft=True) - ax.grid(axis="y", color="#E6EAF0", linewidth=0.8) - ax.axhline(0, color="#536478", linewidth=1.2, linestyle="--") - for index, model in enumerate(models): - if index % 2 == 0: - ax.axvspan(index - 0.5, index + 0.5, facecolor="#F7F9FC", zorder=0) - for offset, variant in enumerate(variants): - x = index + (offset - (len(variants) - 1) / 2) * width - row = lookup.get((arch, model, variant)) - value = row.metrics[metric] if row else math.nan - if not math.isfinite(value): - ax.text(x, padding * 0.05, "N/A", rotation=90, ha="center", va="bottom", - fontsize=8, color="#85909F") - continue - ax.bar(x, value, width=width * 0.9, color=VARIANT_COLORS[variant], - edgecolor="none", linewidth=0, zorder=2) - direction = -1 if value < 0 else 1 - value_label(ax, x, value + direction * padding * 0.05, f"{value:+,.2f}%", - row.validation, direction) - save_figure(fig, f"{mode}_pipeline{pipeline}_{metric}", output_dir) - - -def main() -> None: - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--input", type=Path, default=DEFAULT_RESULTS_PATH) - parser.add_argument("--output-dir", type=Path) - args = parser.parse_args() - if args.output_dir is None: - args.output_dir = args.input.parent / "plots" - configure_style() - results = load_results(args.input) - print(f"Results: {len(results)} configurations, " - f"{sum(row.validation != 'PASS' for row in results)} without PASS validation") - for mode, pipeline in sorted({(row.mode, row.pipeline) for row in results}): - rows = [row for row in results if row.mode == mode and row.pipeline == pipeline] - for metric, label in ( - ("latency", "Single-inference latency" if mode == "latency" else "Output interval"), - ("throughput", "Throughput"), ("power", "Average power"), ("energy", "Energy per inference"), - ): - if metric == "throughput" and mode == "latency": - continue - draw_metric(rows, metric, f"{label} change from baseline (%)", args.output_dir) - - -if __name__ == "__main__": - main() diff --git a/validation/tools/pim/pimcomp/compare/README.md b/validation/tools/pim/pimcomp/compare/README.md index f1c554f..8a67e51 100644 --- a/validation/tools/pim/pimcomp/compare/README.md +++ b/validation/tools/pim/pimcomp/compare/README.md @@ -72,7 +72,7 @@ The comparator defaults to absolute/relative tolerances `1e-3`/`1e-4`; Requires Matplotlib from the root `requirements.txt`. Defaults to the supplied `results_comparison.csv` beside the script and writes `plots/png/` and -`plots/svg/` there. Use `--input PATH` for a runner-generated CSV and +`plots/svg/` and `plots/pdf/` there. Use `--input PATH` for a runner-generated CSV and `--output-dir PATH` to override the destination. The plotter requires the full 40-case matrix: five models × Arch-A/B × latency pipeline 1 and throughput pipelines 2/4/8. Duplicate, missing, unexpected, or inconsistent shared Pimcomp @@ -81,6 +81,9 @@ configurations are rejected before plots are written. | Plot family | Meaning | |---|---| | `validation_status` | Recorded functional status for each compiler and experiment. | +| `validation_status_compact` | Compact functional status matrix. | +| `speedup_compact_*` | Touching single-inference latency and best-pipeline throughput bars for each network. | +| `energy_compact_*`, `power_compact_*` | Energy and power advantages for the same cases selected by the compact speedup plots. | | `latency_*` | Single-inference latency, latency mode only. | | `throughput_*` | Inferences/s, throughput mode only. | | `longest_stage_latency_*` | Combined single-inference latency and throughput output interval; the latter is a stage-latency proxy. | @@ -93,7 +96,28 @@ advantages in either direction. Ratios above 1 favor Raptor: Pimcomp/Raptor for latency, power, and energy; Raptor/Pimcomp for throughput. Diverging bars use winning factor minus one, with parity at zero. -Failed validation is labeled `INVALID`; other non-PASS statuses keep their +The shared `plot_style.py` defines the manuscript palette and display names. +Raptor uses sage green; PIMCOMP uses plain sand. Stage patterns, +outlines, marker shapes, and status text preserve distinctions in grayscale. +Compact plots use blue for the single-inference case and hatched sage +green for the throughput case. +Each architecture/network pair uses the highest recorded throughput across pipelines. +Energy and power use those same cases, without selecting a different pipeline for each metric. +Validation labels remain visible for the selected cases. Units appear in axis +titles and bar values; axis tick labels contain numbers only. + +Throughput-stage bars touch within each network in both absolute and advantage +plots. Bar-label widths are measured before export; dense figures expand only +as needed to contain values and invalidity annotations. Legends use the available +width and balance entries across rows when wrapping is necessary. + +Comparison panels share y-axis titles and tick labels. Each network in the +pipeline plot retains its own scale, with y-axis ticks on both architecture rows. +Only the bottom row shows the shared x-axis ticks. Network names appear above +the top row; architecture names appear on the right. That plot +trims unused vertical range and is intended to span both manuscript columns. + +Failed validation is labeled `P. INVALID`; other non-PASS statuses keep their names, and missing measurements show `N/A`. Relative bars require both compilers to pass to be valid. Repeated Pimcomp batch measurements and statuses must agree before being collapsed. `FAIL` alone does not identify the cause. diff --git a/validation/tools/pim/pimcomp/compare/generate_plots.py b/validation/tools/pim/pimcomp/compare/generate_plots.py index 974ca75..cac9938 100755 --- a/validation/tools/pim/pimcomp/compare/generate_plots.py +++ b/validation/tools/pim/pimcomp/compare/generate_plots.py @@ -1,25 +1,25 @@ #!/usr/bin/env python3 -"""Generate publication-ready Raptor vs Pimcomp comparison charts. +"""Generate publication-ready Raptor vs PIMCOMP comparison charts. Usage: .venv/bin/python validation/tools/pim/pimcomp/compare/generate_plots.py Dependencies: matplotlib (no pandas or seaborn required). By default, reads results_comparison.csv beside this script and writes plots/ -beside it, with outputs grouped in png/ and svg/ under the output directory. +beside it, with outputs grouped in png/, svg/, and pdf/ under the output directory. -YOLO models without Pimcomp measurements get separate Raptor-only charts. +YOLO models without PIMCOMP measurements get separate Raptor-only charts. Comparisons use vertical bars for ratios, diverging advantages, and absolute values, with compiler colors and patterns for the Raptor pipeline stages. Throughput-mode latency is the output interval, a bottleneck-stage proxy, not end-to-end inference latency. Separate throughput plots use inferences/s. Energy is reported per inference; power is averaged over the simulation. Numerical results whose compiler did not pass functional validation have a red -status following their value label; missing Pimcomp +status following their value label; missing PIMCOMP measurements appear as 'N/A'. The fixed compiler palette is shared by -all metrics. Relative plots use Pimcomp / Raptor for latency, power and energy, -and Raptor / Pimcomp for throughput, so >1 favors Raptor. Diverging plots put the winner's -factor above (Raptor) or below (Pimcomp) parity, with 1x at the zero line. +all metrics. Relative plots use PIMCOMP / Raptor for latency, power and energy, +and Raptor / PIMCOMP for throughput, so >1 favors Raptor. Diverging plots put the winner's +factor above (Raptor) or below (PIMCOMP) parity, with 1x at the zero line. """ from __future__ import annotations @@ -39,20 +39,22 @@ from matplotlib.lines import Line2D from matplotlib.ticker import FuncFormatter, MultipleLocator sys.path.insert(0, str(Path(__file__).resolve().parents[2])) -from plot_style import COLORS, HATCH_COLOR, TEXT_COLOR, configure_style, fit_value_labels, save_figure, value_label +from plot_style import (ARCH_NAMES, BAND_COLOR, COMPILER_NAMES, COMPILER_STYLES, + ERROR_COLOR, LINE_COLOR, MATRIX_TEXT_COLOR, METRIC_STYLES, + MODE_NAMES, MODEL_NAMES, NETWORK_STYLES, + PALETTE, TEXT_COLOR, add_legend, configure_style, fit_value_labels, + fit_bar_widths, fit_x_tick_labels, save_figure, value_label) -COMPILER_COLORS = {"raptor": COLORS[0], "pimcomp": COLORS[2]} STAGE_PATTERNS = {1: "", 2: "..", 4: "//", 8: "\\\\"} ARCHITECTURES = ("arch-a", "arch-b") MODELS = ("vgg8", "resnet18", "resnet34", "googlenet", "yolo11n") CONFIGURATIONS = (("latency", 1), ("throughput", 2), ("throughput", 4), ("throughput", 8)) -LINE_COLOR = "#A8B2C0" VALIDATION_STYLES = { - "PASS": ("#168C8B", "white", ""), - "FAIL": ("#D18D32", HATCH_COLOR, ""), - "SKIPPED": ("#E7DFF0", TEXT_COLOR, ""), - "NA": ("#E8EDF1", TEXT_COLOR, ""), + "PASS": PALETTE["sage"], + "FAIL": PALETTE["red"], + "SKIPPED": PALETTE["pale_sand"], + "NA": PALETTE["pale_blue"], } @@ -85,6 +87,10 @@ class Result: def throughput_speedup(self) -> float: return quotient(self.raptor_throughput, self.pimcomp_throughput) + @property + def speedup(self) -> float: + return self.latency_speedup if self.mode == "latency" else self.throughput_speedup + def positive_float(text: str | None) -> float: try: @@ -129,7 +135,7 @@ def load_results(path: Path) -> list[Result]: seen[key] = line schedule = row["pimcomp_pipeline"].strip().lower() if schedule != ("element" if mode == "latency" else "batch"): - raise ValueError(f"{key}: incompatible Pimcomp schedule {schedule!r}") + raise ValueError(f"{key}: incompatible PIMCOMP schedule {schedule!r}") rows.append(Result( arch=arch, model=model, @@ -185,20 +191,20 @@ def figure_legend(fig, results: list[Result], field: str, for _, compiler, value in available} - {None} if len(visible_compilers) > 1: for compiler in compilers: - handles.append(Patch(facecolor=COMPILER_COLORS[compiler], - label=compiler.capitalize() + ("" if absolute else " advantage"))) + fill, border = COMPILER_STYLES[compiler] + handles.append(Patch(facecolor=fill, edgecolor=border, + label=COMPILER_NAMES[compiler] + ("" if absolute else " advantage"))) handles += [Patch(facecolor="white", edgecolor=TEXT_COLOR, hatch=STAGE_PATTERNS[depth], - label=f"{depth} stage" + (" (latency)" if depth == 1 else "s")) + label=("1 stage (latency)" if absolute else "Single-inference") + if depth == 1 else f"{depth} stages") for depth in sorted(stages) if stages != {1}] styles = [] if len(available) < len(measurements): - styles.append(Patch(facecolor="white", edgecolor="#B2BAC5", + styles.append(Patch(facecolor="white", edgecolor=LINE_COLOR, label="N/A: missing measurement")) handles += styles if handles: - fig.legend(handles=handles, loc="outside upper center", ncol=(len(handles) + 1) // 2, - frameon=False, fontsize=9.5, handlelength=1.7, columnspacing=1.5, - handletextpad=0.6, labelspacing=0.4) + add_legend(fig, handles) def absolute_columns(results: list[Result], raptor_only: bool = False) -> list[tuple[Result, str]]: @@ -218,7 +224,7 @@ def absolute_columns(results: list[Result], raptor_only: bool = False) -> list[t for row in throughput } if len(signatures) != 1: - raise ValueError(f"{arch}/{model}: inconsistent shared Pimcomp batch results across pipelines") + raise ValueError(f"{arch}/{model}: inconsistent shared PIMCOMP batch results across pipelines") columns.append((throughput[0], "pimcomp")) return columns @@ -231,34 +237,50 @@ def label_groups(ax, results: list[Result], positions: list[float] | None = None for index, (model, rows) in enumerate(groupby(results, key=lambda row: row.model)): end = start + len(list(rows)) centers.append((positions[start] + positions[end - 1]) / 2) - labels.append(model.upper()) + labels.append(MODEL_NAMES[model]) left = ax.get_xlim()[0] if start == 0 else (positions[start - 1] + positions[start]) / 2 right = ax.get_xlim()[1] if end == len(results) else (positions[end - 1] + positions[end]) / 2 if index % 2 == 0: - ax.axvspan(left, right, facecolor="#F7F9FC", zorder=0) + ax.axvspan(left, right, facecolor=BAND_COLOR, zorder=0) if index > 0: - ax.axvline(left, color="#DDE3EB", linewidth=0.9) + ax.axvline(left, color=LINE_COLOR, linewidth=0.7) start = end ax.set_xticks(centers, labels, fontsize=9.5) +def grouped_positions(results: list[Result]) -> list[float]: + positions = [] + for index, row in enumerate(results): + if index == 0: + positions.append(0.0) + continue + previous = results[index - 1] + gap = 0.8 if row.model != previous.model else 0.4 if row.mode != previous.mode else 0.0 + positions.append(positions[-1] + 1.0 + gap) + return positions + + def diverging_ratio(ratio: float) -> float: return ratio - 1 if ratio >= 1 else 1 - 1 / ratio -def draw_validation(results: list[Result], output_dir: Path) -> None: - columns = absolute_columns(results) - configurations = list(dict.fromkeys((row.mode, row.pipeline if compiler == "raptor" else row.pimcomp_schedule, - compiler) for row, compiler in columns)) +def draw_validation(results: list[Result], output_dir: Path, compact: bool = False) -> None: + cells = {} + for row, compiler in absolute_columns(results): + stage = row.pipeline if compiler == "raptor" else row.pimcomp_schedule + key = (row.arch, row.model, row.mode, None if compact else stage, compiler) + cells.setdefault(key, set()).add(getattr(row, f"{compiler}_validation")) + configurations = list(dict.fromkeys(key[2:] for key in cells)) positions = [] for index, (mode, _, _) in enumerate(configurations): gap = 0.4 if index and mode != configurations[index - 1][0] else 0 positions.append(positions[-1] + 1 + gap if index else 0) configuration_x = dict(zip(configurations, positions)) - headers = [f"Raptor\n{stage} stages" if mode == "throughput" and compiler == "raptor" - else "Pimcomp\nbatch" if mode == "throughput" else compiler.capitalize() + headers = [f"Raptor\n{stage} stages" if mode == "throughput" and compiler == "raptor" and not compact + else "PIMCOMP\nbatch" if mode == "throughput" and compiler == "pimcomp" and not compact + else COMPILER_NAMES[compiler] for mode, stage, compiler in configurations] - fig, axes = plt.subplots(1, 2, figsize=(14, 3.8), layout="constrained") + fig, axes = plt.subplots(1, 2, figsize=(8.5 if compact else 12, 3.3), layout="constrained") for ax, arch in zip(axes, ARCHITECTURES): ax.set_xlim(positions[0] - 0.55, positions[-1] + 0.55) ax.set_ylim(len(MODELS) - 0.5, -0.5) @@ -266,31 +288,41 @@ def draw_validation(results: list[Result], output_dir: Path) -> None: for mode, group in groupby(zip(configurations, positions), key=lambda item: item[0][0]): group_positions = [x for _, x in group] centers.append((group_positions[0] + group_positions[-1]) / 2) - modes.append(mode.capitalize()) + modes.append("Latency" if mode == "latency" else MODE_NAMES[mode]) ax.set_xticks(centers, modes, fontsize=10) ax.set_xticks(positions, headers, minor=True) ax.tick_params(axis="x", which="minor", top=True, labeltop=True, bottom=False, labelbottom=False, length=0, pad=6, labelsize=9) - ax.set_yticks(range(len(MODELS)), [model.upper() for model in MODELS], fontsize=9) + ax.set_yticks(range(len(MODELS)), [MODEL_NAMES[model] for model in MODELS], fontsize=9) ax.tick_params(axis="both", length=0, pad=6) - architecture = "ARCHITECTURE A" if arch == "arch-a" else "ARCHITECTURE B" + ax.tick_params(axis="y", labelleft=arch == ARCHITECTURES[0]) + architecture = ARCH_NAMES[arch] ax.set_xlabel(architecture, fontsize=11, weight="bold", labelpad=8) - for row, compiler in columns: - if row.arch != arch: + for (cell_arch, model, mode, stage, compiler), statuses in cells.items(): + if cell_arch != arch: continue - key = (row.mode, row.pipeline if compiler == "raptor" else row.pimcomp_schedule, compiler) - x, y = configuration_x[key], MODELS.index(row.model) - status = getattr(row, f"{compiler}_validation") - color, text_color, _ = VALIDATION_STYLES[status] + x, y = configuration_x[(mode, stage, compiler)], MODELS.index(model) + status = next(status for status in ("FAIL", "SKIPPED", "NA", "PASS") if status in statuses) + color, border = VALIDATION_STYLES[status] ax.add_patch(Rectangle((x - 0.5, y - 0.42), 1, 0.84, - facecolor=color, edgecolor="#657081", linewidth=0.5)) + facecolor=color, edgecolor=border, linewidth=0.7)) ax.text(x, y, "N/A" if status == "NA" else status, - color=text_color, ha="center", va="center", fontsize=9) - save_figure(fig, "validation_status", output_dir) + color=MATRIX_TEXT_COLOR, ha="center", va="center", fontsize=12, weight="normal") + fit_x_tick_labels(fig) + save_figure(fig, "validation_status_compact" if compact else "validation_status", output_dir) + + +def select_compact_speedups(results: list[Result]) -> list[Result]: + selected = [] + for _, group in groupby(results, key=lambda row: (row.arch, row.model, row.mode)): + rows = list(group) + selected.append(max(rows, key=lambda row: row.speedup if math.isfinite(row.speedup) else -math.inf)) + return selected def draw_metric(results: list[Result], field: str, ylabel: str, - basename: str, output_dir: Path, diverging: bool = False) -> None: + basename: str, output_dir: Path, diverging: bool = False, + compact: bool = False) -> None: data = [diverging_ratio(getattr(row, field)) if diverging else getattr(row, field) for row in results] available = [v for v in data if math.isfinite(v)] @@ -301,49 +333,68 @@ def draw_metric(results: list[Result], field: str, ylabel: str, y_min = math.floor((lower - padding) * 2) / 2 if diverging else 0 y_interval = 1.0 if y_max - y_min > 4.5 else 0.5 - fig, axes = plt.subplots(1, 2, sharey=True, figsize=(14, 5.2), + fig, axes = plt.subplots(1, 2, sharey=True, figsize=(8.5 if compact else 12, 4.4), layout="constrained") - figure_legend(fig, results, field) + if compact: + single_metric = "latency" if field == "speedup" else field + throughput_suffix = "" if field == "speedup" else f" {field}" + handles = [Patch(facecolor=METRIC_STYLES["latency"][0], + edgecolor=METRIC_STYLES["latency"][1], + label=f"Single-inference {single_metric}"), + Patch(facecolor=METRIC_STYLES["throughput"][0], + edgecolor=METRIC_STYLES["throughput"][1], hatch="//", + label=f"Best-case throughput{throughput_suffix}")] for ax, arch in zip(axes, ARCHITECTURES): selected = [row for row in results if row.arch == arch] - ax.set_xlabel("ARCHITECTURE A" if arch == "arch-a" else "ARCHITECTURE B", + positions = [index + index // 2 for index in range(len(selected))] if compact else grouped_positions(selected) + ax.set_xlabel(ARCH_NAMES[arch], fontsize=11, weight="bold", labelpad=8) ax.set_axisbelow(True) ax.set_ylim(y_min, y_max) - ax.set_xlim(-0.5, len(selected) - 0.5) - label_groups(ax, selected) - ax.tick_params(axis="y", labelleft=True, pad=5, length=0) + ax.set_xlim(-0.9, positions[-1] + 0.9 if positions else 0.9) + label_groups(ax, selected, positions) + ax.tick_params(axis="y", labelleft=arch == ARCHITECTURES[0], pad=5, length=0) ax.tick_params(axis="x", labelsize=9.5, length=0, pad=6) ax.yaxis.set_major_locator(MultipleLocator(y_interval)) if diverging: - ax.yaxis.set_major_formatter(FuncFormatter(lambda value, _: f"{abs(value) + 1:g}×")) - ax.grid(axis="y", color="#E6EAF0", linewidth=0.8) - ax.axhline(0 if diverging else 1, color="#536478", linewidth=1.2, + ax.yaxis.set_major_formatter(FuncFormatter(lambda value, _: f"{abs(value) + 1:g}")) + ax.grid(axis="y") + ax.axhline(0 if diverging else 1, color=TEXT_COLOR, linewidth=1.0, linestyle="--", zorder=3) - ax.set_ylabel(ylabel, fontsize=10.5, labelpad=6) + if arch == ARCHITECTURES[0]: + ax.set_ylabel(ylabel, fontsize=10.5, labelpad=6) - for position, row in enumerate(selected): + for position, row in zip(positions, selected): ratio = getattr(row, field) if not math.isfinite(ratio): ax.text(position, (y_max - y_min) * 0.02, "N/A", ha="center", va="bottom", - fontsize=9.5, color="#85909F", fontstyle="italic") + fontsize=9.5, color=TEXT_COLOR, fontstyle="italic") continue compiler = advantage_compiler(ratio) statuses = [status for status in (row.raptor_validation, row.pimcomp_validation) if status != "PASS"] status = "FAIL" if "FAIL" in statuses else next(iter(statuses), "PASS") pattern = STAGE_PATTERNS[row.pipeline] + color, border = COMPILER_STYLES[compiler] if compiler else ("white", TEXT_COLOR) + if compact: + pattern = "//" if row.mode == "throughput" else "" + color, border = METRIC_STYLES[row.mode] measurement = diverging_ratio(ratio) if diverging else ratio - ax.bar(position, measurement, width=0.72, - color=COMPILER_COLORS.get(compiler, "#657081"), + ax.bar(position, measurement, width=1.0, + color=color, hatch=pattern, - edgecolor=TEXT_COLOR if pattern else "none", - linewidth=0.7 if pattern else 0.0, zorder=2) + edgecolor=border, linewidth=0.7, zorder=2) direction = -1 if measurement < 0 else 1 factor = max(ratio, 1 / ratio) if diverging else ratio value_label(ax, position, measurement + direction * (y_max - y_min) * 0.012, - f"{factor:.2f}×", status, direction, fontsize=9.1) + f"{factor:.2f}×", status, direction, fontsize=9.1, invalid_label="P. INVALID") + fit_bar_widths(fig) + if compact: + add_legend(fig, handles) + else: + figure_legend(fig, results, field) + fit_x_tick_labels(fig) fit_value_labels(fig) save_figure(fig, basename, output_dir) @@ -356,21 +407,13 @@ def draw_absolute(results: list[Result], field: str, ylabel: str, for row, compiler in columns if math.isfinite(getattr(row, f"{compiler}_{field}"))] y_max = max(available, default=1.0) * 1.35 - fig, axes = plt.subplots(1, 2, sharey=True, figsize=(14, 5.2), + fig, axes = plt.subplots(1, 2, sharey=True, figsize=(12, 4.4), layout="constrained") - figure_legend(fig, results, field, absolute=True, raptor_only=raptor_only) for ax, arch in zip(axes, ARCHITECTURES): selected = [(row, compiler) for row, compiler in columns if row.arch == arch] - positions = [] - for index, (row, _) in enumerate(selected): - if index == 0: - positions.append(0.0) - continue - previous = selected[index - 1][0] - gap = 0.8 if row.model != previous.model else 0.4 if row.mode != previous.mode else 0.0 - positions.append(positions[-1] + 1.0 + gap) - ax.set_xlabel("ARCHITECTURE A" if arch == "arch-a" else "ARCHITECTURE B", + positions = grouped_positions([row for row, _ in selected]) + ax.set_xlabel(ARCH_NAMES[arch], fontsize=11, weight="bold", labelpad=8) ax.set_axisbelow(True) ax.set_ylim(0, y_max) @@ -384,14 +427,15 @@ def draw_absolute(results: list[Result], field: str, ylabel: str, for (_, mode), group in groupby(enumerate(selected), key=lambda item: (item[1][0].model, item[1][0].mode)): group_positions = [positions[position] for position, _ in group] centers.append((group_positions[0] + group_positions[-1]) / 2) - labels.append(mode.capitalize()) + labels.append("Latency" if mode == "latency" else MODE_NAMES[mode]) ax.set_xticks(centers, labels, minor=True) ax.tick_params(axis="x", which="minor", length=0, pad=4, labelsize=8) - ax.set_ylabel(ylabel, fontsize=10.5) + if arch == ARCHITECTURES[0]: + ax.set_ylabel(ylabel, fontsize=10.5) if field == "throughput": ax.ticklabel_format(axis="y", style="plain", useOffset=False) - ax.grid(axis="y", color="#E6EAF0", linewidth=0.8) - ax.tick_params(axis="y", length=0, pad=6) + ax.grid(axis="y") + ax.tick_params(axis="y", labelleft=arch == ARCHITECTURES[0], length=0, pad=6) ax.tick_params(axis="x", which="major", length=0, pad=22 if {row.mode for row, _ in selected} == {"latency", "throughput"} else 6) @@ -399,19 +443,22 @@ def draw_absolute(results: list[Result], field: str, ylabel: str, measurement = getattr(row, f"{compiler}_{field}") * scale if not math.isfinite(measurement): ax.text(x, y_max * 0.02, "N/A", rotation=90, ha="center", - va="bottom", fontsize=8, color="#85909F") + va="bottom", fontsize=8, color=TEXT_COLOR) continue status = getattr(row, f"{compiler}_validation") pattern = STAGE_PATTERNS[row.pipeline] if compiler == "raptor" else "" + fill, border = COMPILER_STYLES[compiler] ax.bar(x, measurement, width=1.0, - color=COMPILER_COLORS[compiler], + color=fill, hatch=pattern, - edgecolor=TEXT_COLOR if pattern else "none", - linewidth=0.7 if pattern else 0.0, zorder=2) + edgecolor=border, linewidth=0.7, zorder=2) label = (f"{measurement:,.2f}".rstrip("0").rstrip(".") if field == "throughput" else f"{measurement:.3g}") - value_label(ax, x, measurement + y_max * 0.012, label, status) + value_label(ax, x, measurement + y_max * 0.012, label, status, invalid_label="P. INVALID") + fit_bar_widths(fig) + figure_legend(fig, results, field, absolute=True, raptor_only=raptor_only) + fit_x_tick_labels(fig) fit_value_labels(fig) save_figure(fig, basename, output_dir) @@ -419,43 +466,50 @@ def draw_absolute(results: list[Result], field: str, ylabel: str, def draw_pipeline_effect(results: list[Result], output_dir: Path) -> None: rows = [row for row in results if row.mode == "throughput" or row.pipeline == 1] stages = sorted({row.pipeline for row in rows}) - panels = [(model, color) for model, color in zip(MODELS, COLORS) - if any(row.model == model for row in rows)] + panels = [model for model in MODELS if any(row.model == model for row in rows)] lookup = {(row.arch, row.model, row.pipeline): row for row in rows} rates = {key: quotient(1000, row.raptor_latency) if row.mode == "latency" else row.raptor_throughput for key, row in lookup.items()} fig, axes = plt.subplots(len(ARCHITECTURES), len(panels), sharex=True, sharey="col", - figsize=(3 * len(panels), 7.2), squeeze=False, layout="constrained") - for column, (model, color) in enumerate(panels): + figsize=(2.6 * len(panels), 4.3), squeeze=False, layout="constrained") + fig.supylabel("Throughput (inferences/s)", fontsize=10.5) + for column, model in enumerate(panels): + fill, border = NETWORK_STYLES[model] for index, arch in enumerate(ARCHITECTURES): ax = axes[index, column] keys = [(arch, model, stage) for stage in stages] values = [rates.get(key, math.nan) for key in keys] valid = [key in lookup and lookup[key].raptor_validation == "PASS" for key in keys] - ax.plot(stages, values, color=color, linewidth=1.8) + ax.plot(stages, values, color=border, linewidth=1.8) for stage, value, passed in zip(stages, values, valid): if math.isfinite(value): marker = ("s" if stage == 1 else "o") if passed else "x" - ax.plot(stage, value, color=color, marker=marker, + status = lookup[(arch, model, stage)].raptor_validation + ax.plot(stage, value, color=ERROR_COLOR if status == "FAIL" else border, + markerfacecolor=fill, marker=marker, markersize=6, linestyle="none") ax.set_xticks(stages) - ax.set_xlabel(model.upper(), fontsize=10.5, weight="bold", labelpad=8) - ax.grid(axis="y", color="#E6EAF0", linewidth=0.8) - ax.tick_params(axis="both", length=0, pad=6, labelbottom=True, labelleft=True) + if index == 0: + ax.set_title(MODEL_NAMES[model], fontsize=10.5, weight="normal", pad=8) + ax.grid(axis="y") + ax.tick_params(axis="both", length=0, pad=6, + labelbottom=index == len(ARCHITECTURES) - 1, labelleft=True) ax.ticklabel_format(axis="y", style="plain", useOffset=False) - if column == 0: - ax.set_ylabel(f"{arch.replace('arch-', 'ARCHITECTURE ').upper()}\n" - "Throughput (inferences/s)", fontsize=10.5) + if column == len(panels) - 1: + ax.yaxis.set_label_position("right") + ax.set_ylabel(ARCH_NAMES[arch], fontsize=10.5, weight="bold", rotation=270, labelpad=24) for ax in axes.flat: - ax.set_ylim(bottom=0) - handles = [Line2D([], [], color="#657081", marker=marker, linestyle="none", label=label) - for marker, label in (("s", "Derived as 1000ms / latency_ms"), + ax.margins(y=0.08) + handles = [Line2D([], [], color=TEXT_COLOR, markerfacecolor="white", + marker=marker, linestyle="none", label=label) + for marker, label in (("s", "Derived as 1 / latency"), ("o", "Throughput measurement"))] if any(row.raptor_validation != "PASS" and math.isfinite(rates[key]) for key, row in lookup.items()): - handles.append(Line2D([], [], color="#424B58", marker="x", linestyle="none", + handles.append(Line2D([], [], color=TEXT_COLOR, marker="x", linestyle="none", label="Raptor validation not PASS")) - fig.legend(handles=handles, loc="outside upper center", ncol=len(handles), frameon=False, fontsize=9.5) + add_legend(fig, handles) fig.supxlabel("Raptor pipeline stages", fontsize=11) + fit_x_tick_labels(fig) save_figure(fig, "raptor_pipeline_throughput", output_dir) @@ -471,6 +525,7 @@ def main() -> None: print(f"Results: {len(results)} configurations, {valid} valid comparisons, " f"{len(results)-valid} comparisons without two PASS statuses") draw_validation(results, args.output_dir) + draw_validation(results, args.output_dir, compact=True) draw_pipeline_effect(results, args.output_dir) yolo_only_models = { model for model in MODELS if model.startswith("yolo") @@ -480,12 +535,19 @@ def main() -> None: } comparisons = [row for row in results if row.model not in yolo_only_models] yolo_only = [row for row in results if row.model in yolo_only_models] + compact_speedups = select_compact_speedups(comparisons) + for metric, label in (("speedup", "Speedup (x)"), + ("energy", "Energy-per-inference advantage (x)"), + ("power", "Average power advantage (x)")): + for suffix, diverging in (("comparison", False), ("diverging", True)): + draw_metric(compact_speedups, metric, label, + f"{metric}_compact_{suffix}", args.output_dir, diverging=diverging, compact=True) for metric, name, label, mode in [ ("latency_speedup", "longest_stage_latency", "Longest stage latency speedup (x)", None), ("latency_speedup", "latency", "Single-inference latency speedup (x)", "latency"), ("throughput_speedup", "throughput", "Throughput speedup (x)", "throughput"), ("power", "power", "Average power advantage (x)", None), - ("energy", "energy", "Energy per inference advantage (x)", None), + ("energy", "energy", "Energy-per-inference advantage (x)", None), ]: selected = [row for row in comparisons if mode is None or row.mode == mode] draw_metric(selected, metric, label, diff --git a/validation/tools/pim/plot_style.py b/validation/tools/pim/plot_style.py index 88d3500..d97fc1b 100644 --- a/validation/tools/pim/plot_style.py +++ b/validation/tools/pim/plot_style.py @@ -1,4 +1,4 @@ -"""Shared appearance and PNG/SVG export for PIM result plots.""" +"""Shared appearance and PNG/SVG/PDF export for PIM result plots.""" from pathlib import Path from math import ceil @@ -12,9 +12,32 @@ from matplotlib.patheffects import AbstractPathEffect from matplotlib.transforms import Affine2D, IdentityTransform, TransformedPath -TEXT_COLOR = "#243449" -HATCH_COLOR = "#C62828" -COLORS = ("#2B65A1", "#168C8B", "#D18D32", "#8062A5", "#B65D60") +PALETTE = { + "sage": ("#B3D5CA", "#4B7C6A"), + "sand": ("#E1BC8B", "#856A44"), + "pale_blue": ("#EDF3FA", "#456B90"), + "blue": ("#B3CCE3", "#456B90"), + "pale_green": ("#EAF5F1", "#3E8271"), + "pale_sand": ("#FCF1E5", "#A87940"), + "red": ("#FF9494", "#C52020"), + "purple": ("#D1C0E0", "#77558F"), +} +TEXT_COLOR = "#222222" +MATRIX_TEXT_COLOR = "#1D2530" +ERROR_COLOR = PALETTE["red"][1] +GRID_COLOR = "#E6E9EC" +LINE_COLOR = "#C9CED3" +BAND_COLOR = "#F7F9FA" +COMPILER_STYLES = {"raptor": PALETTE["sage"], "pimcomp": PALETTE["sand"]} +METRIC_STYLES = {"latency": PALETTE["blue"], "throughput": PALETTE["sage"]} +NETWORK_STYLES = {"vgg8": PALETTE["blue"], "resnet18": PALETTE["sage"], + "resnet34": PALETTE["sand"], "googlenet": PALETTE["purple"], + "yolo11n": PALETTE["red"]} +COMPILER_NAMES = {"raptor": "Raptor", "pimcomp": "PIMCOMP"} +ARCH_NAMES = {"arch-a": "Arch-A", "arch-b": "Arch-B"} +MODEL_NAMES = {"vgg8": "VGG-8", "resnet18": "ResNet-18", "resnet34": "ResNet-34", + "googlenet": "GoogLeNet", "yolo11n": "YOLO11n"} +MODE_NAMES = {"latency": "Single-inference", "throughput": "Throughput"} class CenteredHatch(AbstractPathEffect): @@ -77,51 +100,130 @@ def configure_style() -> None: "text.color": TEXT_COLOR, "xtick.color": TEXT_COLOR, "ytick.color": TEXT_COLOR, - "axes.edgecolor": "#CFD6DF", + "axes.edgecolor": LINE_COLOR, + "grid.color": GRID_COLOR, + "grid.linewidth": 0.8, + "hatch.color": TEXT_COLOR, "savefig.facecolor": "white", "figure.facecolor": "white", }) +def add_legend(fig, handles) -> None: + """Use the fewest balanced legend rows that fit the figure width.""" + renderer = fig.canvas.get_renderer() + for rows in range(1, len(handles) + 1): + columns = ceil(len(handles) / rows) + grid = [] + start = 0 + for row in range(rows): + count = len(handles) // rows + (row < len(handles) % rows) + grid.append(handles[start:start + count] + [None] * (columns - count)) + start += count + ordered = [handle for column in zip(*grid) for handle in column] + legend = fig.legend(handles=[handle if handle is not None else Patch(visible=False) for handle in ordered], + labels=[handle.get_label() if handle is not None else "" for handle in ordered], + loc="outside upper center", ncols=columns, frameon=False, fontsize=9.5, + handlelength=1.7, columnspacing=1.5, handletextpad=0.6, labelspacing=0.4) + for handle, patch, text in zip(ordered, legend.legend_handles, legend.get_texts()): + if handle is None: + patch.set_visible(False) + text.set_visible(False) + if legend.get_window_extent(renderer).width <= fig.bbox.width - 20: + return + legend.remove() + + def value_label(ax, x, y, label: str, status: str = "PASS", direction: int = 1, - fontsize: float = 8) -> None: + fontsize: float = 8, invalid_label: str = "INVALID") -> None: alignment = "bottom" if direction > 0 else "top" rotation = 90 if direction > 0 else -90 - text = ax.text(x, y, label, rotation=rotation, ha="center", va=alignment, - fontsize=fontsize, color=TEXT_COLOR) + offset = direction * fontsize * 0.65 if status != "PASS" else 0 + ax.annotate(label, (x, y), xytext=(-offset, 0), textcoords="offset points", + rotation=rotation, ha="center", va=alignment, fontsize=fontsize, + color=TEXT_COLOR) if status != "PASS": - height = text.get_window_extent(ax.figure.canvas.get_renderer()).height - offset = direction * (height * 72 / ax.figure.dpi + 3) - ax.annotate("INVALID" if status == "FAIL" else status, (x, y), - xytext=(0, offset), textcoords="offset points", + ax.annotate(invalid_label if status == "FAIL" else status, (x, y), + xytext=(offset, 0), textcoords="offset points", rotation=rotation, ha="center", va=alignment, fontsize=fontsize, - color=HATCH_COLOR) + color=ERROR_COLOR if status == "FAIL" else TEXT_COLOR) + + +def fit_x_tick_labels(fig) -> None: + """Rotate only overlapping labels, clockwise by 30 degrees.""" + fig.canvas.draw() + renderer = fig.canvas.get_renderer() + for minor in (False, True): + labels = [(text, text.get_window_extent(renderer)) + for ax in fig.axes for text in ax.get_xticklabels(minor=minor) + if text.get_visible() and text.get_text()] + crowded = set() + for index, (text, bounds) in enumerate(labels): + for other_text, other_bounds in labels[:index]: + if bounds.overlaps(other_bounds): + crowded.update((text, other_text)) + for text in crowded: + text.set_rotation(-30) + text.set_rotation_mode("anchor") + text.set_ha("right" if text.get_va() == "bottom" else "left") + fig.canvas.draw() + renderer = fig.canvas.get_renderer() + for ax in fig.axes: + bottom_minor = [text for text in ax.get_xticklabels(minor=True) + if text.get_va() == "top" and text.get_visible() and text.get_text()] + if bottom_minor: + height = max(text.get_window_extent(renderer).height for text in bottom_minor) + ax.tick_params(axis="x", which="major", pad=height * 72 / fig.dpi + 6) + fig.canvas.draw() + + +def fit_bar_widths(fig) -> None: + """Reserve enough horizontal space for each bar's rendered annotations.""" + fig.canvas.draw() + renderer = fig.canvas.get_renderer() + scale = 1.0 + for ax in fig.axes: + labels = {} + for text in ax.texts: + if hasattr(text, "xy"): + labels.setdefault(text.xy[0], []).append(text.get_window_extent(renderer)) + for container in ax.containers: + for bar in container.patches: + bounds = labels.get(bar.get_x() + bar.get_width() / 2, []) + if bounds: + width = max(box.x1 for box in bounds) - min(box.x0 for box in bounds) + scale = max(scale, (width + renderer.points_to_pixels(2)) / bar.get_window_extent(renderer).width) + if scale > 1: + fig.set_size_inches(fig.get_figwidth() * scale, fig.get_figheight()) def fit_value_labels(fig) -> None: - """Reserve vertical space for rotated labels using their rendered size.""" + """Fit common vertical limits to the data and rendered value labels.""" for ax in fig.axes: for text in ax.texts: text.set_in_layout(False) fig.canvas.draw() renderer = fig.canvas.get_renderer() limits = [] + labels = [] for ax in fig.axes: bounds = ax.get_window_extent(renderer) - labels = [] for text in ax.texts: x, y = text.xy if hasattr(text, "xy") else text.get_position() anchor = ax.transData.transform((x, y))[1] extent = text.get_window_extent(renderer) labels.append((y, max(0, anchor - extent.y0 + 3) / bounds.height, max(0, extent.y1 - anchor + 3) / bounds.height)) - lower, upper = ax.get_ylim() - for _ in range(12): - lower = min([lower, *((y - below * upper) / (1 - below) for y, below, _ in labels)]) - upper = max([upper, *((y - above * lower) / (1 - above) for y, _, above in labels)]) - limits.append((lower, upper)) + limits.append(ax.dataLim.intervaly if ax.has_data() else ax.get_ylim()) + lower = min(0, *(lower for lower, _ in limits)) + upper = max(0, *(upper for _, upper in limits)) + if lower == upper: + upper = lower + 1 + for _ in range(12): + lower = min([lower, *((y - below * upper) / (1 - below) for y, below, _ in labels)]) + upper = max([upper, *((y - above * lower) / (1 - above) for y, _, above in labels)]) for ax in fig.axes: - ax.set_ylim(min(lower for lower, _ in limits), max(upper for _, upper in limits)) + ax.set_ylim(lower, upper) def save_figure(fig, basename: str, output_dir: Path) -> None: @@ -129,7 +231,7 @@ def save_figure(fig, basename: str, output_dir: Path) -> None: hatch = patch.get_hatch() or "" if any(symbol in hatch for symbol in "./\\"): patch.set_path_effects([CenteredHatch()]) - for extension in ("png", "svg"): + for extension in ("png", "svg", "pdf"): target = output_dir / extension / f"{basename}.{extension}" target.parent.mkdir(parents=True, exist_ok=True) fig.savefig(target, dpi=220, bbox_inches="tight", pad_inches=0.04)