From 888fed0591b1dd519d1b998aea5c76ead04a5b31 Mon Sep 17 00:00:00 2001 From: LiangGao Date: Mon, 25 Aug 2025 13:14:15 +0800 Subject: [PATCH 1/7] Change the checker to support threshold for uint16/uint4/int4 --- onnxruntime/test/unittest_util/checkers.cc | 75 ++++++++++++++++++++-- 1 file changed, 70 insertions(+), 5 deletions(-) diff --git a/onnxruntime/test/unittest_util/checkers.cc b/onnxruntime/test/unittest_util/checkers.cc index 7b2a5a4a4ff2f..c2a170768e839 100644 --- a/onnxruntime/test/unittest_util/checkers.cc +++ b/onnxruntime/test/unittest_util/checkers.cc @@ -225,17 +225,27 @@ template <> struct TensorCheck { void operator()(const Tensor& expected, const Tensor& actual, const ValidateOutputParams& params, const std::string& /*provider_type*/) const { - ORT_UNUSED_PARAMETER(params); + const bool has_abs_err = params.absolute_error.has_value(); + Tensor expected_sorted, actual_sorted; const Int4x2* cur_expected; const Int4x2* cur_actual; const auto size = narrow(actual.Shape().Size()); cur_expected = expected.Data(); cur_actual = actual.Data(); + double threshold; + if (has_abs_err) { + threshold = *(params.absolute_error); + } for (size_t i = 0; i < size; ++i) { size_t r = i >> 1; size_t c = i & 0x1; - EXPECT_EQ(cur_expected[r].GetElem(c), cur_actual[r].GetElem(c)) << "i:" << i; + // TODO: the relative error is not used for int4 yet. + if (has_abs_err) { + EXPECT_NEAR(cur_expected[r].GetElem(c), cur_actual[r].GetElem(c), threshold) << "i:" << i; + } else { + EXPECT_EQ(cur_expected[r].GetElem(c), cur_actual[r].GetElem(c)) << "i:" << i; + } } } }; @@ -245,16 +255,28 @@ struct TensorCheck { void operator()(const Tensor& expected, const Tensor& actual, const ValidateOutputParams& params, const std::string& /*provider_type*/) const { ORT_UNUSED_PARAMETER(params); + const bool has_abs_err = params.absolute_error.has_value(); + Tensor expected_sorted, actual_sorted; const UInt4x2* cur_expected; const UInt4x2* cur_actual; const auto size = narrow(actual.Shape().Size()); cur_expected = expected.Data(); cur_actual = actual.Data(); - for (size_t i = 0; i < size; ++i) { + double threshold; + if (has_abs_err) { + threshold = *(params.absolute_error); + } + + for (size_t i = 0; i < static_cast(size); ++i) { size_t r = i >> 1; size_t c = i & 0x1; - EXPECT_EQ(cur_expected[r].GetElem(c), cur_actual[r].GetElem(c)) << "i:" << i; + // TODO: the relative error is not used for int4 yet. + if (has_abs_err) { + EXPECT_NEAR(cur_expected[r].GetElem(c), cur_actual[r].GetElem(c), threshold) << "i:" << i; + } else { + EXPECT_EQ(cur_expected[r].GetElem(c), cur_actual[r].GetElem(c)) << "i:" << i; + } } } }; @@ -292,7 +314,7 @@ struct TensorCheck { // For any other EPs, we still expect an exact match for the results // TODO: Verify if DML can possibly have a ROUNDING_MODE parameter and conform to the other EPs #41968513 if ((provider_type == kNnapiExecutionProvider || provider_type == kDmlExecutionProvider || - provider_type == kXnnpackExecutionProvider) && + provider_type == kXnnpackExecutionProvider || provider_type == kOpenVINOExecutionProvider) && (has_abs_err || has_rel_err)) { double threshold = has_abs_err ? *(params.absolute_error) : 0.0; @@ -357,6 +379,49 @@ struct TensorCheck { } }; +template <> +struct TensorCheck { + void operator()(const Tensor& expected, + const Tensor& actual, + const ValidateOutputParams& params, + const std::string& ) const { + const bool has_abs_err = params.absolute_error.has_value(); + const bool has_rel_err = params.relative_error.has_value(); + + Tensor expected_sorted, actual_sorted; + const uint16_t* cur_expected; + const uint16_t* cur_actual; + const auto size = actual.Shape().Size(); + if (params.sort_output) { + sort_expected_and_actual_buffers(expected, expected_sorted, actual, actual_sorted); + cur_expected = expected_sorted.Data(); + cur_actual = actual_sorted.Data(); + } else { + cur_expected = expected.Data(); + cur_actual = actual.Data(); + } + + if (has_abs_err || has_rel_err) { + double threshold = has_abs_err ? *(params.absolute_error) + : 0.0; + + for (int64_t i = 0; i < size; ++i) { + if (has_rel_err) { + EXPECT_NEAR(cur_expected[i], cur_actual[i], + *(params.relative_error) * cur_expected[i]) // expected[i] is unsigned, can't be negative + << "i:" << i; + } else { // has_abs_err + EXPECT_NEAR(cur_expected[i], cur_actual[i], threshold) << "i:" << i; + } + } + } else { + for (int64_t i = 0; i < size; ++i) { + EXPECT_EQ(cur_expected[i], cur_actual[i]) << "i:" << i; + } + } + } +}; + template <> struct TensorCheck { void operator()(const Tensor& expected, From d1a1897db2e5796797b165659fc8f8c4d9d9e259 Mon Sep 17 00:00:00 2001 From: LiangGao Date: Mon, 25 Aug 2025 13:14:57 +0800 Subject: [PATCH 2/7] Disable loop case for OV-EP due to floating nodes --- onnxruntime/test/providers/cpu/controlflow/loop_test.cc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/onnxruntime/test/providers/cpu/controlflow/loop_test.cc b/onnxruntime/test/providers/cpu/controlflow/loop_test.cc index 07cd2114372dd..0bed6b6e9abee 100644 --- a/onnxruntime/test/providers/cpu/controlflow/loop_test.cc +++ b/onnxruntime/test/providers/cpu/controlflow/loop_test.cc @@ -828,7 +828,8 @@ TEST(Loop, Opset11WithNoVariadicInputsAndOutputs) { test.AddOutput("loop_scan_out", {1}, {1.0f}); // Disable TensorRT on unsupported data type BOOL - test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); + // Disable OpenVino for floating nodes + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider, kOpenVINOExecutionProvider}); } // Test a combination of things: From f5d06b1ef9a72ec0af03f594c1cfc3a7a554a821 Mon Sep 17 00:00:00 2001 From: LiangGao Date: Mon, 25 Aug 2025 13:15:53 +0800 Subject: [PATCH 3/7] Disable the case for OV-EP due to input conflict --- onnxruntime/test/providers/cpu/tensor/slice_op.test.cc | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/onnxruntime/test/providers/cpu/tensor/slice_op.test.cc b/onnxruntime/test/providers/cpu/tensor/slice_op.test.cc index 5b2865a3feed7..f531e2a7210c5 100644 --- a/onnxruntime/test/providers/cpu/tensor/slice_op.test.cc +++ b/onnxruntime/test/providers/cpu/tensor/slice_op.test.cc @@ -540,6 +540,10 @@ TEST(SliceTest, Slice1D_ReverseAllAxes_1) { GTEST_SKIP() << "Skipping because of the following error: Expected output shape [{4}] did not match run output shape [{0}] for output"; } + if (DefaultOpenVINOExecutionProvider().get() != nullptr) { + GTEST_SKIP() << "Skipping because of the following error: The input ends did not supported int max when step is negtive."; + } + RunSliceTest({4}, {1.0f, 2.0f, 3.0f, 4.0f}, {-1}, From 5b2aa1a5f45ff83922a2494280cb8a5491d838f7 Mon Sep 17 00:00:00 2001 From: LiangGao Date: Mon, 25 Aug 2025 13:16:49 +0800 Subject: [PATCH 4/7] Add threshold for uint8/int8/uint16 cases --- onnxruntime/test/contrib_ops/quantize_ops_test.cc | 2 ++ .../test/providers/cpu/tensor/quantize_linear_test.cc | 7 +++++++ onnxruntime/test/providers/cpu/tensor/resize_op_test.cc | 2 ++ 3 files changed, 11 insertions(+) diff --git a/onnxruntime/test/contrib_ops/quantize_ops_test.cc b/onnxruntime/test/contrib_ops/quantize_ops_test.cc index db685967ae5ff..de10f14ef4538 100644 --- a/onnxruntime/test/contrib_ops/quantize_ops_test.cc +++ b/onnxruntime/test/contrib_ops/quantize_ops_test.cc @@ -287,6 +287,7 @@ TEST(QuantizeLinearContribOpTest, QuantizeLinear_per_tensor_float_int8) { 127, -127, 127, -128, 127, -128}); + test.SetOutputAbsErr("y", 1.0f); // Disable Tensorrt EP due to error: node1_quantize_scale_node: out of bounds channel axis 1. Number of input dimensions is 1. test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); } @@ -311,6 +312,7 @@ TEST(QuantizeLinearContribOpTest, QuantizeLinear_per_tensor_float_uint16) { 32769, 32765, 65535, 0, 65535, 0}); + test.SetOutputAbsErr("y", 1.0f); // Disable Tensorrt EP due to error: unsupported data type test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); diff --git a/onnxruntime/test/providers/cpu/tensor/quantize_linear_test.cc b/onnxruntime/test/providers/cpu/tensor/quantize_linear_test.cc index 46acb5a730a78..18eec7d1b42a3 100644 --- a/onnxruntime/test/providers/cpu/tensor/quantize_linear_test.cc +++ b/onnxruntime/test/providers/cpu/tensor/quantize_linear_test.cc @@ -448,6 +448,7 @@ TEST(QuantizeLinearOpTest, Uint16) { 32769, 32765, 65535, 0, 65535, 0}); + test.SetOutputAbsErr("y", 1.0f); // Disable Tensorrt EP due to error: unsupported data type test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); @@ -477,6 +478,7 @@ TEST(QuantizeLinearOpTest, Int16) { 32767, -32768, 32767, -32768, 32767, -32768}); + test.SetOutputAbsErr("y", 1.0f); // Disable Tensorrt EP due to error: unsupported data type test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); @@ -501,6 +503,7 @@ TEST(QuantizeLinearOpTest, Int4) { test.AddOutput("y", dims, {Int4x2(-8, -7), Int4x2(-1, 1), Int4x2(2, 7), Int4x2(7, unused_val)}); + test.SetOutputAbsErr("y", 1.0f); test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); } @@ -568,6 +571,7 @@ TEST(QuantizeLinearOpTest, OddLarge_Int4) { test.AddInput("scale", {}, {scale}, true); test.AddInput("zero_point", {}, {Int4x2(zp, unused_val)}, true); test.AddOutput("y", dims, output); + test.SetOutputAbsErr("y", 1.0f); test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); } @@ -594,6 +598,7 @@ TEST(QuantizeLinearOpTest, OddLarge_UInt4) { test.AddInput("scale", {}, {scale}, true); test.AddInput("zero_point", {}, {UInt4x2(zp, unused_val)}, true); test.AddOutput("y", dims, output); + test.SetOutputAbsErr("y", 1.0f); test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); } @@ -611,6 +616,7 @@ TEST(QuantizeLinearOpTest, Int8_NegativeZeroPoint) { test.AddInput("y_scale", {}, {.039215686f}); test.AddInput("y_zero_point", {}, {-23}); test.AddOutput("y", dims, {-23, 28, 53, 104, 127, -74, -128, -128}); + test.SetOutputAbsErr("y", 1.0f); // Disable Tensorrt EP due to the error, node1_quantize_scale_node: out of bounds channel axis 1. Number of input dimensions is 1. test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); } @@ -628,6 +634,7 @@ TEST(QuantizeLinearOpTest, Int8_PositiveZeroPoint) { test.AddInput("y_scale", {}, {.039215686f}); test.AddInput("y_zero_point", {}, {23}); test.AddOutput("y", dims, {23, 74, 99, 127, 127, -28, -104, -128}); + test.SetOutputAbsErr("y", 1.0f); // Disable Tensorrt EP due to error:node1_quantize_scale_node: out of bounds channel axis 1. Number of input dimensions is 1. test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); } diff --git a/onnxruntime/test/providers/cpu/tensor/resize_op_test.cc b/onnxruntime/test/providers/cpu/tensor/resize_op_test.cc index bb053bc37ce30..f3b0695bdbd9c 100644 --- a/onnxruntime/test/providers/cpu/tensor/resize_op_test.cc +++ b/onnxruntime/test/providers/cpu/tensor/resize_op_test.cc @@ -308,6 +308,7 @@ TEST(ResizeOpTest, NhwcResizeOpLinearDownSampleTest_4DBilinear_uint8) { std::vector Y = {2, 4}; test.AddOutput("Y", {N, static_cast(H * scales[1]), static_cast(W * scales[2]), C}, Y); + test.SetOutputAbsErr("Y", 1.0f); // CUDA: result mismatch due to not implementing NHWC support // ROCm: results mismatch test.Run(OpTester::ExpectResult::kExpectSuccess, "", @@ -647,6 +648,7 @@ TEST(ResizeOpTest, NhwcResizeOpLinearDownSampleTest_4DBilinear_pytorch_half_pixe std::vector Y = {1, 7, 12}; test.AddOutput("Y", {N, sizes[1], sizes[2], C}, Y); + test.SetOutputAbsErr("Y", 1.0f); // CUDA: result mismatch due to not implementing NHWC support // ROCm: results mismatch // DML: results mismatch From fc6c1af53e160e585ba3b0487ce82d282c4dc81b Mon Sep 17 00:00:00 2001 From: LiangGao Date: Mon, 25 Aug 2025 14:33:49 +0800 Subject: [PATCH 5/7] Fix build warning --- onnxruntime/test/unittest_util/checkers.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/onnxruntime/test/unittest_util/checkers.cc b/onnxruntime/test/unittest_util/checkers.cc index c2a170768e839..1b64ca5addf52 100644 --- a/onnxruntime/test/unittest_util/checkers.cc +++ b/onnxruntime/test/unittest_util/checkers.cc @@ -232,7 +232,7 @@ struct TensorCheck { const auto size = narrow(actual.Shape().Size()); cur_expected = expected.Data(); cur_actual = actual.Data(); - double threshold; + double threshold = 0.0f; if (has_abs_err) { threshold = *(params.absolute_error); } @@ -263,7 +263,7 @@ struct TensorCheck { cur_expected = expected.Data(); cur_actual = actual.Data(); - double threshold; + double threshold = 0.0f; if (has_abs_err) { threshold = *(params.absolute_error); } From 43433d148eb46262843e48e0701570fd329f2f2f Mon Sep 17 00:00:00 2001 From: LiangGao Date: Tue, 2 Sep 2025 15:45:36 +0800 Subject: [PATCH 6/7] New changes after rebase --- onnxruntime/test/providers/cpu/tensor/cast_op_test.cc | 3 +++ onnxruntime/test/providers/cpu/tensor/concat_op_test.cc | 2 ++ 2 files changed, 5 insertions(+) diff --git a/onnxruntime/test/providers/cpu/tensor/cast_op_test.cc b/onnxruntime/test/providers/cpu/tensor/cast_op_test.cc index 8f4c4ff0896ba..8f2eac2d05792 100644 --- a/onnxruntime/test/providers/cpu/tensor/cast_op_test.cc +++ b/onnxruntime/test/providers/cpu/tensor/cast_op_test.cc @@ -853,6 +853,9 @@ TEST(CastOpTest, Int32ToInt4x2OddNumberOfElements) { } TEST(CastOpTest, Int32ToInt4x2EmptyTensor) { + if (DefaultOpenVINOExecutionProvider().get() != nullptr) { + GTEST_SKIP() << "The OpenVINO not support 0 size input"; + } // GIVEN const std::vector empty_shape{0}; const std::vector empty_input = {}; diff --git a/onnxruntime/test/providers/cpu/tensor/concat_op_test.cc b/onnxruntime/test/providers/cpu/tensor/concat_op_test.cc index b5e13c6377ccb..5f08b6df6785d 100644 --- a/onnxruntime/test/providers/cpu/tensor/concat_op_test.cc +++ b/onnxruntime/test/providers/cpu/tensor/concat_op_test.cc @@ -73,6 +73,7 @@ TEST(ConcatOpTest, Concat1D_2) { test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider, // TensorRT: no support for dynamic shape tensor kNnapiExecutionProvider, // NNAPI: concat does not support 0 size input + kOpenVINOExecutionProvider, // OpenVINO: does not support 0 size input kQnnExecutionProvider}); // QNN: not support dynamic shape tensor } @@ -118,6 +119,7 @@ TEST(ConcatOpTest, Concat2D_3) { test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider, // TensorRT: no support for dynamic shape tensor kNnapiExecutionProvider, // NNAPI: concat does not support 0 size input + kOpenVINOExecutionProvider, // OpenVINO: does not support 0 size input kQnnExecutionProvider}); // QNN: not support dynamic shape tensor } From 13fecd8543935b49759e49f61ac1820320b9d2ab Mon Sep 17 00:00:00 2001 From: LiangGao Date: Tue, 2 Sep 2025 15:47:29 +0800 Subject: [PATCH 7/7] Change code based on review --- onnxruntime/test/providers/cpu/tensor/slice_op.test.cc | 2 +- onnxruntime/test/unittest_util/checkers.cc | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/onnxruntime/test/providers/cpu/tensor/slice_op.test.cc b/onnxruntime/test/providers/cpu/tensor/slice_op.test.cc index f531e2a7210c5..657f3fe9c127a 100644 --- a/onnxruntime/test/providers/cpu/tensor/slice_op.test.cc +++ b/onnxruntime/test/providers/cpu/tensor/slice_op.test.cc @@ -541,7 +541,7 @@ TEST(SliceTest, Slice1D_ReverseAllAxes_1) { } if (DefaultOpenVINOExecutionProvider().get() != nullptr) { - GTEST_SKIP() << "Skipping because of the following error: The input ends did not supported int max when step is negtive."; + GTEST_SKIP() << "Skipping because of the following error: The input ends do not support int max when step is negative."; } RunSliceTest({4}, diff --git a/onnxruntime/test/unittest_util/checkers.cc b/onnxruntime/test/unittest_util/checkers.cc index 1b64ca5addf52..794bd24310cd1 100644 --- a/onnxruntime/test/unittest_util/checkers.cc +++ b/onnxruntime/test/unittest_util/checkers.cc @@ -254,7 +254,6 @@ template <> struct TensorCheck { void operator()(const Tensor& expected, const Tensor& actual, const ValidateOutputParams& params, const std::string& /*provider_type*/) const { - ORT_UNUSED_PARAMETER(params); const bool has_abs_err = params.absolute_error.has_value(); Tensor expected_sorted, actual_sorted; const UInt4x2* cur_expected;