integer overflow fixed in getContinuousSize()

savuor · savuor · commit be7c924e7cd7 · 2016-09-02T01:50:54.000+03:00
diff --git a/modules/core/src/precomp.hpp b/modules/core/src/precomp.hpp
@@ -127,39 +127,46 @@ template<typename T> struct OpMax
     T operator ()(const T a, const T b) const { return std::max(a, b); }
 };
 
-inline Size getContinuousSize( const Mat& m1, int widthScale=1 )
+inline Size getContinuousSize_(int flags, int cols, int rows, int widthScale)
 {
-    return m1.isContinuous() ? Size(m1.cols*m1.rows*widthScale, 1) :
-        Size(m1.cols*widthScale, m1.rows);
+    int64 sz = (int64)cols * rows * widthScale;
+    return (flags & Mat::CONTINUOUS_FLAG) != 0 &&
+        (int)sz == sz ? Size((int)sz, 1) : Size(cols * widthScale, rows);
 }
 
-inline Size getContinuousSize( const Mat& m1, const Mat& m2, int widthScale=1 )
+inline Size getContinuousSize(const Mat& m1, int widthScale = 1)
 {
-    return (m1.flags & m2.flags & Mat::CONTINUOUS_FLAG) != 0 ?
-        Size(m1.cols*m1.rows*widthScale, 1) : Size(m1.cols*widthScale, m1.rows);
+    return getContinuousSize_(m1.flags,
+                              m1.cols, m1.rows, widthScale);
 }
 
-inline Size getContinuousSize( const Mat& m1, const Mat& m2,
-                               const Mat& m3, int widthScale=1 )
+inline Size getContinuousSize(const Mat& m1, const Mat& m2, int widthScale = 1)
 {
-    return (m1.flags & m2.flags & m3.flags & Mat::CONTINUOUS_FLAG) != 0 ?
-        Size(m1.cols*m1.rows*widthScale, 1) : Size(m1.cols*widthScale, m1.rows);
+    return getContinuousSize_(m1.flags & m2.flags,
+                              m1.cols, m1.rows, widthScale);
 }
 
-inline Size getContinuousSize( const Mat& m1, const Mat& m2,
-                               const Mat& m3, const Mat& m4,
-                               int widthScale=1 )
+inline Size getContinuousSize(const Mat& m1, const Mat& m2,
+                              const Mat& m3, int widthScale = 1)
 {
-    return (m1.flags & m2.flags & m3.flags & m4.flags & Mat::CONTINUOUS_FLAG) != 0 ?
-        Size(m1.cols*m1.rows*widthScale, 1) : Size(m1.cols*widthScale, m1.rows);
+    return getContinuousSize_(m1.flags & m2.flags & m3.flags,
+                              m1.cols, m1.rows, widthScale);
 }
 
-inline Size getContinuousSize( const Mat& m1, const Mat& m2,
-                               const Mat& m3, const Mat& m4,
-                               const Mat& m5, int widthScale=1 )
+inline Size getContinuousSize(const Mat& m1, const Mat& m2,
+                              const Mat& m3, const Mat& m4,
+                              int widthScale = 1)
 {
-    return (m1.flags & m2.flags & m3.flags & m4.flags & m5.flags & Mat::CONTINUOUS_FLAG) != 0 ?
-        Size(m1.cols*m1.rows*widthScale, 1) : Size(m1.cols*widthScale, m1.rows);
+    return getContinuousSize_(m1.flags & m2.flags & m3.flags & m4.flags,
+                              m1.cols, m1.rows, widthScale);
+}
+
+inline Size getContinuousSize(const Mat& m1, const Mat& m2,
+                              const Mat& m3, const Mat& m4,
+                              const Mat& m5, int widthScale = 1)
+{
+    return getContinuousSize_(m1.flags & m2.flags & m3.flags & m4.flags & m5.flags,
+                              m1.cols, m1.rows, widthScale);
 }
 
 struct NoVec
