core: fix infinite recursion in compare

sovrasov · sovrasov · commit 35a1ecef2a7d · 2017-06-28T15:00:52.000+03:00
diff --git a/modules/core/src/arithm.cpp b/modules/core/src/arithm.cpp
@@ -1239,15 +1239,18 @@ void cv::compare(InputArray _src1, InputArray _src2, OutputArray _dst, int op)
             || !_src1.sameSize(_src2)
             || _src1.type() != _src2.type())
     {
-        if (checkScalar(_src1, _src2.type(), _src1.kind(), _src2.kind()))
+        bool is_src1_scalar = checkScalar(_src1, _src2.type(), _src1.kind(), _src2.kind());
+        bool is_src2_scalar = checkScalar(_src2, _src1.type(), _src2.kind(), _src1.kind());
+
+        if (is_src1_scalar && !is_src2_scalar)
         {
             op = op == CMP_LT ? CMP_GT : op == CMP_LE ? CMP_GE :
                 op == CMP_GE ? CMP_LE : op == CMP_GT ? CMP_LT : op;
             // src1 is a scalar; swap it with src2
             compare(_src2, _src1, _dst, op);
             return;
         }
-        else if( !checkScalar(_src2, _src1.type(), _src2.kind(), _src1.kind()) )
+        else if( (is_src1_scalar && is_src2_scalar) || (!is_src1_scalar && !is_src2_scalar) )
             CV_Error( CV_StsUnmatchedSizes,
                      "The operation is neither 'array op array' (where arrays have the same size and the same type), "
                      "nor 'array op scalar', nor 'scalar op array'" );
diff --git a/modules/core/test/test_arithm.cpp b/modules/core/test/test_arithm.cpp
@@ -1922,3 +1922,13 @@ TEST(Compare, empty)
     EXPECT_TRUE(dst1.empty());
     EXPECT_TRUE(dst2.empty());
 }
+
+TEST(Compare, regression_8999)
+{
+    Mat_<double> A(4,1); A << 1, 3, 2, 4;
+    Mat_<double> B(1,1); B << 2;
+    Mat C;
+    ASSERT_ANY_THROW({
+                        compare(A, B, C, CMP_LT);
+                     });
+}
