From 1eb9c3dd7a27a5242d5c902b0f4d5f2482d8ac71 Mon Sep 17 00:00:00 2001 From: Scott Todd Date: Fri, 1 Mar 2024 16:35:37 -0800 Subject: [PATCH] Fix XFAIL behavior in conftest.py. --- .github/workflows/test_iree.yml | 2 +- iree_tests/configs/config_cpu_llvm_sync.json | 134 +++------------ iree_tests/configs/config_gpu_vulkan.json | 162 +++++-------------- iree_tests/conftest.py | 22 ++- iree_tests/requirements.txt | 1 + 5 files changed, 83 insertions(+), 238 deletions(-) diff --git a/.github/workflows/test_iree.yml b/.github/workflows/test_iree.yml index 364395bcc..87a0c40bc 100644 --- a/.github/workflows/test_iree.yml +++ b/.github/workflows/test_iree.yml @@ -57,4 +57,4 @@ jobs: - name: "Testing with pytest" run: | source ${VENV_DIR}/bin/activate - pytest iree_tests -n auto -rpfE + pytest iree_tests -n auto -rpfE --timeout=30 diff --git a/iree_tests/configs/config_cpu_llvm_sync.json b/iree_tests/configs/config_cpu_llvm_sync.json index 9b37c8c0f..5745e1b8c 100644 --- a/iree_tests/configs/config_cpu_llvm_sync.json +++ b/iree_tests/configs/config_cpu_llvm_sync.json @@ -6,8 +6,13 @@ "iree_run_module_flags": [ "--device=local-sync" ], - "skip_compile_tests": [], - "skip_run_tests": [], + "skip_compile_tests": [ + // Timeouts? + "test_dequantizelinear", + "test_slice_default_axes", + ], + "skip_run_tests": [ + ], "expected_compile_failures": [ "test_acos_example", "test_acos", @@ -81,8 +86,6 @@ "test_bernoulli_expanded", "test_bernoulli_seed_expanded", "test_bernoulli_seed", - "test_blackmanwindow_expanded", - "test_blackmanwindow_symmetric_expanded", "test_blackmanwindow_symmetric", "test_blackmanwindow", "test_cast_FLOAT_to_FLOAT8E4M3FN", @@ -190,7 +193,6 @@ "test_dequantizelinear_e5m2", "test_dequantizelinear_int16", "test_dequantizelinear_uint16", - "test_dequantizelinear", "test_det_2d", "test_det_nd", "test_dft_axis_opset19", @@ -218,28 +220,15 @@ "test_einsum_transpose", "test_equal_string_broadcast", "test_equal_string", - "test_expand_dim_changed", - "test_expand_dim_unchanged", "test_eyelike_populate_off_main_diagonal", "test_eyelike_with_dtype", "test_eyelike_without_dtype", - "test_flatten_axis0", - "test_flatten_axis1", - "test_flatten_axis2", - "test_flatten_axis3", - "test_flatten_default_axis", - "test_flatten_negative_axis1", - "test_flatten_negative_axis2", - "test_flatten_negative_axis3", - "test_flatten_negative_axis4", "test_gathernd_example_float32", "test_gathernd_example_int32_batch_dim1", "test_gathernd_example_int32", "test_gelu_tanh_1", "test_gelu_tanh_2", "test_gemm_default_no_bias", - "test_globalaveragepool_precomputed", - "test_globalaveragepool", "test_globalmaxpool_precomputed", "test_globalmaxpool", "test_greater_bcast", @@ -270,12 +259,8 @@ "test_gru_defaults", "test_gru_seq_length", "test_gru_with_initial_bias", - "test_hammingwindow_expanded", - "test_hammingwindow_symmetric_expanded", "test_hammingwindow_symmetric", "test_hammingwindow", - "test_hannwindow_expanded", - "test_hannwindow_symmetric_expanded", "test_hannwindow_symmetric", "test_hannwindow", "test_hardmax_axis_0", @@ -296,14 +281,6 @@ "test_image_decoder_decode_pnm_rgb", "test_image_decoder_decode_tiff_rgb", "test_image_decoder_decode_webp_rgb", - "test_instancenorm_epsilon", - "test_instancenorm_example", - "test_isinf_float16", - "test_isinf_negative", - "test_isinf_positive", - "test_isinf", - "test_isnan_float16", - "test_isnan", "test_layer_normalization_2d_axis_negative_1_expanded_ver18", "test_layer_normalization_2d_axis_negative_1_expanded", "test_layer_normalization_2d_axis_negative_2_expanded_ver18", @@ -342,26 +319,12 @@ "test_layer_normalization_4d_axis3_expanded", "test_layer_normalization_default_axis_expanded_ver18", "test_layer_normalization_default_axis_expanded", - "test_logsoftmax_axis_0_expanded_ver18", - "test_logsoftmax_axis_0_expanded", "test_logsoftmax_axis_0", - "test_logsoftmax_axis_1_expanded_ver18", - "test_logsoftmax_axis_1_expanded", "test_logsoftmax_axis_1", - "test_logsoftmax_axis_2_expanded_ver18", - "test_logsoftmax_axis_2_expanded", "test_logsoftmax_axis_2", - "test_logsoftmax_default_axis_expanded_ver18", - "test_logsoftmax_default_axis_expanded", "test_logsoftmax_default_axis", - "test_logsoftmax_example_1_expanded_ver18", - "test_logsoftmax_example_1_expanded", "test_logsoftmax_example_1", - "test_logsoftmax_large_number_expanded_ver18", - "test_logsoftmax_large_number_expanded", "test_logsoftmax_large_number", - "test_logsoftmax_negative_axis_expanded_ver18", - "test_logsoftmax_negative_axis_expanded", "test_logsoftmax_negative_axis", "test_loop11", "test_lppool_1d_default", @@ -381,28 +344,14 @@ "test_matmulinteger", "test_max_one_input", "test_maxpool_1d_default", - "test_maxpool_2d_ceil", - "test_maxpool_2d_default", - "test_maxpool_2d_dilations", - "test_maxpool_2d_pads", - "test_maxpool_2d_precomputed_pads", "test_maxpool_2d_precomputed_same_upper", - "test_maxpool_2d_precomputed_strides", "test_maxpool_2d_same_lower", "test_maxpool_2d_same_upper", - "test_maxpool_2d_strides", "test_maxpool_2d_uint8", - "test_maxpool_3d_default", - "test_maxpool_3d_dilations_use_ref_impl_large", - "test_maxpool_3d_dilations_use_ref_impl", - "test_maxpool_3d_dilations", "test_maxpool_with_argmax_2d_precomputed_pads", "test_maxpool_with_argmax_2d_precomputed_strides", "test_maxunpool_export_with_output_shape", "test_maxunpool_export_without_output_shape", - "test_mean_example", - "test_mean_one_input", - "test_mean_two_inputs", "test_melweightmatrix", "test_min_one_input", "test_mish_expanded", @@ -422,11 +371,9 @@ "test_mod_uint8", "test_momentum_multiple", "test_momentum", - "test_mvn_expanded_ver18", "test_mvn_expanded", "test_mvn", "test_nesterov_momentum", - "test_nllloss_NC_expanded", "test_nllloss_NC", "test_nllloss_NCd1_expanded", "test_nllloss_NCd1_ii_expanded", @@ -438,14 +385,12 @@ "test_nllloss_NCd1_weight_ii", "test_nllloss_NCd1_weight", "test_nllloss_NCd1", - "test_nllloss_NCd1d2_expanded", "test_nllloss_NCd1d2_no_weight_reduction_mean_ii_expanded", "test_nllloss_NCd1d2_no_weight_reduction_mean_ii", "test_nllloss_NCd1d2_reduction_mean_expanded", "test_nllloss_NCd1d2_reduction_mean", "test_nllloss_NCd1d2_reduction_sum_expanded", "test_nllloss_NCd1d2_reduction_sum", - "test_nllloss_NCd1d2_with_weight_expanded", "test_nllloss_NCd1d2_with_weight_reduction_mean_expanded", "test_nllloss_NCd1d2_with_weight_reduction_mean", "test_nllloss_NCd1d2_with_weight_reduction_sum_expanded", @@ -454,13 +399,11 @@ "test_nllloss_NCd1d2_with_weight_reduction_sum", "test_nllloss_NCd1d2_with_weight", "test_nllloss_NCd1d2", - "test_nllloss_NCd1d2d3_none_no_weight_negative_ii_expanded", "test_nllloss_NCd1d2d3_none_no_weight_negative_ii", "test_nllloss_NCd1d2d3_sum_weight_high_ii_expanded", "test_nllloss_NCd1d2d3_sum_weight_high_ii", "test_nllloss_NCd1d2d3d4d5_mean_weight_expanded", "test_nllloss_NCd1d2d3d4d5_mean_weight", - "test_nllloss_NCd1d2d3d4d5_none_no_weight_expanded", "test_nllloss_NCd1d2d3d4d5_none_no_weight", "test_nonmaxsuppression_center_point_box_format", "test_nonmaxsuppression_flipped_coordinates", @@ -485,9 +428,6 @@ "test_pow_types_int32_int32", "test_pow_types_int64_float32", "test_pow_types_int64_int64", - "test_prelu_broadcast", - "test_prelu_example", - "test_qlinearconv", "test_qlinearmatmul_2D_int8_float16", "test_qlinearmatmul_2D_int8_float32", "test_qlinearmatmul_2D_uint8_float16", @@ -502,11 +442,8 @@ "test_quantizelinear_e5m2", "test_quantizelinear_int16", "test_quantizelinear_uint16", - "test_quantizelinear", "test_range_float_type_positive_delta_expanded", - "test_range_float_type_positive_delta", "test_range_int32_type_negative_delta_expanded", - "test_range_int32_type_negative_delta", "test_reduce_l1_default_axes_keepdims_example", "test_reduce_l1_default_axes_keepdims_random", "test_reduce_l1_do_not_keepdims_example_expanded", @@ -629,15 +566,6 @@ "test_regex_full_match_email_domain", "test_regex_full_match_empty", "test_reshape_allowzero_reordered", - "test_reshape_extended_dims", - "test_reshape_negative_dim", - "test_reshape_negative_extended_dims", - "test_reshape_one_dim", - "test_reshape_reduced_dims", - "test_reshape_reordered_all_dims", - "test_reshape_reordered_last_dims", - "test_reshape_zero_and_negative_dim", - "test_reshape_zero_dim", "test_resize_downsample_scales_cubic_A_n0p5_exclude_outside", "test_resize_downsample_scales_cubic_align_corners", "test_resize_downsample_scales_cubic_antialias", @@ -776,7 +704,6 @@ "test_sinh", "test_size_example", "test_size", - "test_slice_default_axes", "test_slice_default_steps", "test_slice_end_out_of_bounds", "test_slice_neg_steps", @@ -784,19 +711,12 @@ "test_slice_negative_axes", "test_slice_start_out_of_bounds", "test_slice", - "test_softmax_axis_0_expanded_ver18", "test_softmax_axis_0_expanded", - "test_softmax_axis_1_expanded_ver18", "test_softmax_axis_1_expanded", - "test_softmax_axis_2_expanded_ver18", "test_softmax_axis_2_expanded", - "test_softmax_default_axis_expanded_ver18", "test_softmax_default_axis_expanded", - "test_softmax_example_expanded_ver18", "test_softmax_example_expanded", - "test_softmax_large_number_expanded_ver18", "test_softmax_large_number_expanded", - "test_softmax_negative_axis_expanded_ver18", "test_softmax_negative_axis_expanded", "test_softplus_example", "test_softplus", @@ -894,7 +814,7 @@ "test_unsqueeze_two_axes", "test_unsqueeze_unsorted_axes", "test_upsample_nearest", - "test_wrap_pad" + "test_wrap_pad", ], "expected_run_failures": [ "test_argmax_default_axis_example", @@ -940,40 +860,34 @@ "test_cast_BFLOAT16_to_FLOAT", "test_cast_DOUBLE_to_FLOAT", "test_cast_DOUBLE_to_FLOAT16", - "test_cast_FLOAT16_to_DOUBLE", "test_cast_FLOAT_to_BFLOAT16", "test_cast_FLOAT_to_DOUBLE", - "test_castlike_BFLOAT16_to_FLOAT", + "test_cast_FLOAT16_to_DOUBLE", "test_castlike_BFLOAT16_to_FLOAT_expanded", + "test_castlike_BFLOAT16_to_FLOAT", + "test_castlike_DOUBLE_to_FLOAT_expanded", "test_castlike_DOUBLE_to_FLOAT", - "test_castlike_DOUBLE_to_FLOAT16", "test_castlike_DOUBLE_to_FLOAT16_expanded", - "test_castlike_DOUBLE_to_FLOAT_expanded", - "test_castlike_FLOAT16_to_DOUBLE", - "test_castlike_FLOAT16_to_DOUBLE_expanded", - "test_castlike_FLOAT_to_BFLOAT16", + "test_castlike_DOUBLE_to_FLOAT16", "test_castlike_FLOAT_to_BFLOAT16_expanded", - "test_castlike_FLOAT_to_DOUBLE", + "test_castlike_FLOAT_to_BFLOAT16", "test_castlike_FLOAT_to_DOUBLE_expanded", + "test_castlike_FLOAT_to_DOUBLE", + "test_castlike_FLOAT16_to_DOUBLE_expanded", + "test_castlike_FLOAT16_to_DOUBLE", "test_clip_default_int8_max_expanded", - "test_clip_default_int8_min", "test_clip_default_int8_min_expanded", - "test_convtranspose", - "test_convtranspose_1d", - "test_convtranspose_3d", - "test_convtranspose_dilations", - "test_convtranspose_pad", - "test_convtranspose_pads", + "test_clip_default_int8_min", "test_elu_default", "test_gather_0", "test_gather_1", "test_gather_2d_indices", "test_gather_elements_negative_indices", - "test_gather_negative_indices", - "test_hardsigmoid", "test_hardsigmoid_default", "test_hardsigmoid_example", + "test_hardsigmoid", "test_hardswish_expanded", + "test_logsoftmax_large_number_expanded", "test_max_float64", "test_max_int16", "test_max_int32", @@ -993,18 +907,22 @@ "test_min_uint64", "test_min_uint8", "test_mul_uint8", - "test_pow", + "test_nllloss_NCd1d2_with_weight_expanded", "test_pow_example", "test_pow_types_float32_int32", "test_pow_types_float32_int64", "test_pow_types_float32_uint32", "test_pow_types_float32_uint64", + "test_pow", + "test_qlinearconv", + "test_quantizelinear", + "test_range_int32_type_negative_delta", "test_scatter_elements_with_negative_indices", "test_selu_default", - "test_shape", "test_shape_clip_end", "test_shape_clip_start", "test_shape_example", - "test_where_long_example" + "test_shape", + "test_where_long_example", ] } diff --git a/iree_tests/configs/config_gpu_vulkan.json b/iree_tests/configs/config_gpu_vulkan.json index f59b60da4..a8ad1582c 100644 --- a/iree_tests/configs/config_gpu_vulkan.json +++ b/iree_tests/configs/config_gpu_vulkan.json @@ -6,7 +6,12 @@ "iree_run_module_flags": [ "--device=vulkan" ], - "skip_compile_tests": [], + "skip_compile_tests": [ + // Timeouts? + "test_dequantizelinear", + "test_slice_default_axes", + "test_sum_example", + ], "skip_run_tests": [], "expected_compile_failures": [ "test_acos_example", @@ -81,8 +86,6 @@ "test_bernoulli_expanded", "test_bernoulli_seed_expanded", "test_bernoulli_seed", - "test_blackmanwindow_expanded", - "test_blackmanwindow_symmetric_expanded", "test_blackmanwindow_symmetric", "test_blackmanwindow", "test_cast_FLOAT_to_FLOAT8E4M3FN", @@ -190,7 +193,6 @@ "test_dequantizelinear_e5m2", "test_dequantizelinear_int16", "test_dequantizelinear_uint16", - "test_dequantizelinear", "test_det_2d", "test_det_nd", "test_dft_axis_opset19", @@ -218,28 +220,15 @@ "test_einsum_transpose", "test_equal_string_broadcast", "test_equal_string", - "test_expand_dim_changed", - "test_expand_dim_unchanged", "test_eyelike_populate_off_main_diagonal", "test_eyelike_with_dtype", "test_eyelike_without_dtype", - "test_flatten_axis0", - "test_flatten_axis1", - "test_flatten_axis2", - "test_flatten_axis3", - "test_flatten_default_axis", - "test_flatten_negative_axis1", - "test_flatten_negative_axis2", - "test_flatten_negative_axis3", - "test_flatten_negative_axis4", "test_gathernd_example_float32", "test_gathernd_example_int32_batch_dim1", "test_gathernd_example_int32", "test_gelu_tanh_1", "test_gelu_tanh_2", "test_gemm_default_no_bias", - "test_globalaveragepool_precomputed", - "test_globalaveragepool", "test_globalmaxpool_precomputed", "test_globalmaxpool", "test_greater_bcast", @@ -270,12 +259,8 @@ "test_gru_defaults", "test_gru_seq_length", "test_gru_with_initial_bias", - "test_hammingwindow_expanded", - "test_hammingwindow_symmetric_expanded", "test_hammingwindow_symmetric", "test_hammingwindow", - "test_hannwindow_expanded", - "test_hannwindow_symmetric_expanded", "test_hannwindow_symmetric", "test_hannwindow", "test_hardmax_axis_0", @@ -296,14 +281,6 @@ "test_image_decoder_decode_pnm_rgb", "test_image_decoder_decode_tiff_rgb", "test_image_decoder_decode_webp_rgb", - "test_instancenorm_epsilon", - "test_instancenorm_example", - "test_isinf_float16", - "test_isinf_negative", - "test_isinf_positive", - "test_isinf", - "test_isnan_float16", - "test_isnan", "test_layer_normalization_2d_axis_negative_1_expanded_ver18", "test_layer_normalization_2d_axis_negative_1_expanded", "test_layer_normalization_2d_axis_negative_2_expanded_ver18", @@ -342,26 +319,12 @@ "test_layer_normalization_4d_axis3_expanded", "test_layer_normalization_default_axis_expanded_ver18", "test_layer_normalization_default_axis_expanded", - "test_logsoftmax_axis_0_expanded_ver18", - "test_logsoftmax_axis_0_expanded", "test_logsoftmax_axis_0", - "test_logsoftmax_axis_1_expanded_ver18", - "test_logsoftmax_axis_1_expanded", "test_logsoftmax_axis_1", - "test_logsoftmax_axis_2_expanded_ver18", - "test_logsoftmax_axis_2_expanded", "test_logsoftmax_axis_2", - "test_logsoftmax_default_axis_expanded_ver18", - "test_logsoftmax_default_axis_expanded", "test_logsoftmax_default_axis", - "test_logsoftmax_example_1_expanded_ver18", - "test_logsoftmax_example_1_expanded", "test_logsoftmax_example_1", - "test_logsoftmax_large_number_expanded_ver18", - "test_logsoftmax_large_number_expanded", "test_logsoftmax_large_number", - "test_logsoftmax_negative_axis_expanded_ver18", - "test_logsoftmax_negative_axis_expanded", "test_logsoftmax_negative_axis", "test_loop11", "test_lppool_1d_default", @@ -381,28 +344,14 @@ "test_matmulinteger", "test_max_one_input", "test_maxpool_1d_default", - "test_maxpool_2d_ceil", - "test_maxpool_2d_default", - "test_maxpool_2d_dilations", - "test_maxpool_2d_pads", - "test_maxpool_2d_precomputed_pads", "test_maxpool_2d_precomputed_same_upper", - "test_maxpool_2d_precomputed_strides", "test_maxpool_2d_same_lower", "test_maxpool_2d_same_upper", - "test_maxpool_2d_strides", "test_maxpool_2d_uint8", - "test_maxpool_3d_default", - "test_maxpool_3d_dilations_use_ref_impl_large", - "test_maxpool_3d_dilations_use_ref_impl", - "test_maxpool_3d_dilations", "test_maxpool_with_argmax_2d_precomputed_pads", "test_maxpool_with_argmax_2d_precomputed_strides", "test_maxunpool_export_with_output_shape", "test_maxunpool_export_without_output_shape", - "test_mean_example", - "test_mean_one_input", - "test_mean_two_inputs", "test_melweightmatrix", "test_min_one_input", "test_mish_expanded", @@ -422,11 +371,9 @@ "test_mod_uint8", "test_momentum_multiple", "test_momentum", - "test_mvn_expanded_ver18", "test_mvn_expanded", "test_mvn", "test_nesterov_momentum", - "test_nllloss_NC_expanded", "test_nllloss_NC", "test_nllloss_NCd1_expanded", "test_nllloss_NCd1_ii_expanded", @@ -438,14 +385,12 @@ "test_nllloss_NCd1_weight_ii", "test_nllloss_NCd1_weight", "test_nllloss_NCd1", - "test_nllloss_NCd1d2_expanded", "test_nllloss_NCd1d2_no_weight_reduction_mean_ii_expanded", "test_nllloss_NCd1d2_no_weight_reduction_mean_ii", "test_nllloss_NCd1d2_reduction_mean_expanded", "test_nllloss_NCd1d2_reduction_mean", "test_nllloss_NCd1d2_reduction_sum_expanded", "test_nllloss_NCd1d2_reduction_sum", - "test_nllloss_NCd1d2_with_weight_expanded", "test_nllloss_NCd1d2_with_weight_reduction_mean_expanded", "test_nllloss_NCd1d2_with_weight_reduction_mean", "test_nllloss_NCd1d2_with_weight_reduction_sum_expanded", @@ -454,13 +399,11 @@ "test_nllloss_NCd1d2_with_weight_reduction_sum", "test_nllloss_NCd1d2_with_weight", "test_nllloss_NCd1d2", - "test_nllloss_NCd1d2d3_none_no_weight_negative_ii_expanded", "test_nllloss_NCd1d2d3_none_no_weight_negative_ii", "test_nllloss_NCd1d2d3_sum_weight_high_ii_expanded", "test_nllloss_NCd1d2d3_sum_weight_high_ii", "test_nllloss_NCd1d2d3d4d5_mean_weight_expanded", "test_nllloss_NCd1d2d3d4d5_mean_weight", - "test_nllloss_NCd1d2d3d4d5_none_no_weight_expanded", "test_nllloss_NCd1d2d3d4d5_none_no_weight", "test_nonmaxsuppression_center_point_box_format", "test_nonmaxsuppression_flipped_coordinates", @@ -485,9 +428,6 @@ "test_pow_types_int32_int32", "test_pow_types_int64_float32", "test_pow_types_int64_int64", - "test_prelu_broadcast", - "test_prelu_example", - "test_qlinearconv", "test_qlinearmatmul_2D_int8_float16", "test_qlinearmatmul_2D_int8_float32", "test_qlinearmatmul_2D_uint8_float16", @@ -502,11 +442,8 @@ "test_quantizelinear_e5m2", "test_quantizelinear_int16", "test_quantizelinear_uint16", - "test_quantizelinear", "test_range_float_type_positive_delta_expanded", - "test_range_float_type_positive_delta", "test_range_int32_type_negative_delta_expanded", - "test_range_int32_type_negative_delta", "test_reduce_l1_default_axes_keepdims_example", "test_reduce_l1_default_axes_keepdims_random", "test_reduce_l1_do_not_keepdims_example_expanded", @@ -629,15 +566,6 @@ "test_regex_full_match_email_domain", "test_regex_full_match_empty", "test_reshape_allowzero_reordered", - "test_reshape_extended_dims", - "test_reshape_negative_dim", - "test_reshape_negative_extended_dims", - "test_reshape_one_dim", - "test_reshape_reduced_dims", - "test_reshape_reordered_all_dims", - "test_reshape_reordered_last_dims", - "test_reshape_zero_and_negative_dim", - "test_reshape_zero_dim", "test_resize_downsample_scales_cubic_A_n0p5_exclude_outside", "test_resize_downsample_scales_cubic_align_corners", "test_resize_downsample_scales_cubic_antialias", @@ -776,7 +704,6 @@ "test_sinh", "test_size_example", "test_size", - "test_slice_default_axes", "test_slice_default_steps", "test_slice_end_out_of_bounds", "test_slice_neg_steps", @@ -784,19 +711,12 @@ "test_slice_negative_axes", "test_slice_start_out_of_bounds", "test_slice", - "test_softmax_axis_0_expanded_ver18", "test_softmax_axis_0_expanded", - "test_softmax_axis_1_expanded_ver18", "test_softmax_axis_1_expanded", - "test_softmax_axis_2_expanded_ver18", "test_softmax_axis_2_expanded", - "test_softmax_default_axis_expanded_ver18", "test_softmax_default_axis_expanded", - "test_softmax_example_expanded_ver18", "test_softmax_example_expanded", - "test_softmax_large_number_expanded_ver18", "test_softmax_large_number_expanded", - "test_softmax_negative_axis_expanded_ver18", "test_softmax_negative_axis_expanded", "test_softplus_example", "test_softplus", @@ -900,7 +820,7 @@ "test_min_uint16", "test_max_uint16", "test_min_uint8", - "test_max_uint8" + "test_max_uint8", ], "expected_run_failures": [ "test_argmax_default_axis_example", @@ -946,40 +866,34 @@ "test_cast_BFLOAT16_to_FLOAT", "test_cast_DOUBLE_to_FLOAT", "test_cast_DOUBLE_to_FLOAT16", - "test_cast_FLOAT16_to_DOUBLE", "test_cast_FLOAT_to_BFLOAT16", "test_cast_FLOAT_to_DOUBLE", - "test_castlike_BFLOAT16_to_FLOAT", + "test_cast_FLOAT16_to_DOUBLE", "test_castlike_BFLOAT16_to_FLOAT_expanded", + "test_castlike_BFLOAT16_to_FLOAT", + "test_castlike_DOUBLE_to_FLOAT_expanded", "test_castlike_DOUBLE_to_FLOAT", - "test_castlike_DOUBLE_to_FLOAT16", "test_castlike_DOUBLE_to_FLOAT16_expanded", - "test_castlike_DOUBLE_to_FLOAT_expanded", - "test_castlike_FLOAT16_to_DOUBLE", - "test_castlike_FLOAT16_to_DOUBLE_expanded", - "test_castlike_FLOAT_to_BFLOAT16", + "test_castlike_DOUBLE_to_FLOAT16", "test_castlike_FLOAT_to_BFLOAT16_expanded", - "test_castlike_FLOAT_to_DOUBLE", + "test_castlike_FLOAT_to_BFLOAT16", "test_castlike_FLOAT_to_DOUBLE_expanded", + "test_castlike_FLOAT_to_DOUBLE", + "test_castlike_FLOAT16_to_DOUBLE_expanded", + "test_castlike_FLOAT16_to_DOUBLE", "test_clip_default_int8_max_expanded", - "test_clip_default_int8_min", "test_clip_default_int8_min_expanded", - "test_convtranspose", - "test_convtranspose_1d", - "test_convtranspose_3d", - "test_convtranspose_dilations", - "test_convtranspose_pad", - "test_convtranspose_pads", + "test_clip_default_int8_min", "test_elu_default", "test_gather_0", "test_gather_1", "test_gather_2d_indices", "test_gather_elements_negative_indices", - "test_gather_negative_indices", - "test_hardsigmoid", "test_hardsigmoid_default", "test_hardsigmoid_example", + "test_hardsigmoid", "test_hardswish_expanded", + "test_logsoftmax_large_number_expanded", "test_max_float64", "test_max_int16", "test_max_int32", @@ -999,49 +913,53 @@ "test_min_uint64", "test_min_uint8", "test_mul_uint8", - "test_pow", + "test_nllloss_NCd1d2_with_weight_expanded", "test_pow_example", "test_pow_types_float32_int32", "test_pow_types_float32_int64", "test_pow_types_float32_uint32", "test_pow_types_float32_uint64", + "test_pow", + "test_qlinearconv", + "test_quantizelinear", + "test_range_int32_type_negative_delta", "test_scatter_elements_with_negative_indices", "test_selu_default", - "test_shape", "test_shape_clip_end", "test_shape_clip_start", "test_shape_example", + "test_shape", "test_where_long_example", // These pass on CPU but fail on Vulkan. - "test_castlike_FLOAT_to_FLOAT16", - "test_layer_normalization_4d_axis_negative_2", - "test_cast_FLOAT_to_FLOAT16", - "test_castlike_FLOAT_to_FLOAT16_expanded", "test_and_bcast3v1d", - "test_castlike_FLOAT16_to_FLOAT", + "test_and_bcast4v2d", "test_and_bcast4v4d", + "test_cast_FLOAT_to_FLOAT16", "test_cast_FLOAT16_to_FLOAT", + "test_castlike_FLOAT_to_FLOAT16_expanded", + "test_castlike_FLOAT_to_FLOAT16", + "test_castlike_FLOAT16_to_FLOAT_expanded", + "test_castlike_FLOAT16_to_FLOAT", + "test_isinf_float16", + "test_isnan_float16", "test_max_float16", - "test_ceil", + "test_min_float16", + "test_or_bcast3v1d", "test_or_bcast4v2d", - "test_castlike_FLOAT16_to_FLOAT_expanded", + "test_or_bcast4v4d", "test_reduce_l1_default_axes_keepdims_example_expanded", - "test_min_float16", "test_reduce_l1_default_axes_keepdims_random_expanded", + "test_reduce_mean_default_axes_keepdims_example", "test_reduce_mean_default_axes_keepdims_random", + "test_reduce_sum_default_axes_keepdims_example", + "test_reduce_sum_default_axes_keepdims_random", "test_reduce_sum_empty_axes_input_noop_example", "test_reduce_sum_negative_axes_keepdims_random", - "test_reduce_sum_default_axes_keepdims_random", - "test_reduce_sum_default_axes_keepdims_example", "test_reduce_sum_square_default_axes_keepdims_example_expanded", - "test_or_bcast3v1d", - "test_reduce_mean_default_axes_keepdims_example", "test_reduce_sum_square_default_axes_keepdims_random_expanded", - "test_and_bcast4v2d", - "test_or_bcast4v4d", - "test_xor_bcast4v4d", "test_xor_bcast3v1d", - "test_xor_bcast4v2d" + "test_xor_bcast4v2d", + "test_xor_bcast4v4d", ] } diff --git a/iree_tests/conftest.py b/iree_tests/conftest.py index 64772df22..5cab26957 100644 --- a/iree_tests/conftest.py +++ b/iree_tests/conftest.py @@ -124,19 +124,27 @@ def __init__(self, spec, **kwargs): self.run_args.append(f"--flagfile={self.spec.data_flagfile_name}") def runtest(self): - # First test compilation... if not self.spec.expect_compile_success: - pytest.xfail("Expected compilation to fail") + self.add_marker( + pytest.mark.xfail( + raises=IreeCompileException, + strict=True, + reason="Expected compilation to fail", + ) + ) self.test_compile() - if not self.spec.expect_compile_success: - return - if self.spec.skip_run: + if not self.spec.expect_compile_success or self.spec.skip_run: return - # ... then test runtime execution if not self.spec.expect_run_success: - pytest.xfail("Expected run to fail") + self.add_marker( + pytest.mark.xfail( + raises=IreeRunException, + strict=True, + reason="Expected run to fail", + ) + ) self.test_run() def test_compile(self): diff --git a/iree_tests/requirements.txt b/iree_tests/requirements.txt index 5705d6d1a..71584e552 100644 --- a/iree_tests/requirements.txt +++ b/iree_tests/requirements.txt @@ -1,3 +1,4 @@ pytest +pytest-timeout pytest-xdist pyjson5