conva: fix a crash with {u,}{short,int} images (#4055)

kleisauke · web-flow · commit 8e0e8fe51569 · 2024-07-19T18:30:07.000+01:00
{u,}{short,int} images should use {u,}short as an intermediate type when the length of the longest line is below 256. Resolves: #1162.
diff --git a/ChangeLog b/ChangeLog
@@ -7,6 +7,7 @@
 - fix pipe read limit
 - fix a rare crash on Windows in highly threaded applications [Julianiolo]
 - vipssave: fix infinite loop on Windows with large images [pdbourke]
+- conva: fix a crash with {u,}{short,int} images [erdmann]
 - fix vips_image_get_string
 - heifsave: fix lossless mode [kleisauke]
 
diff --git a/libvips/convolution/conva.c b/libvips/convolution/conva.c
@@ -177,7 +177,7 @@ typedef struct {
 
 	/* The horizontal lines we gather. hline[3] writes to band 3 in the
 	 * intermediate image. max_line is the length of the longest hline:
-	 * over 256 and we need to use an int intermediate for 8-bit images.
+	 * over 256 and we need to use an int intermediate for integer images.
 	 */
 	int n_hline;
 	HLine hline[MAX_LINES];
@@ -933,33 +933,39 @@ vips_conva_hgenerate(VipsRegion *out_region,
 			break;
 
 		case VIPS_FORMAT_USHORT:
-			HCONV(unsigned short, unsigned int);
+			if (conva->max_line < 256)
+				HCONV(unsigned short, unsigned short);
+			else
+				HCONV(unsigned short, unsigned int);
 			break;
 
 		case VIPS_FORMAT_SHORT:
-			HCONV(signed short, signed int);
+			if (conva->max_line < 256)
+				HCONV(signed short, signed short);
+			else
+				HCONV(signed short, signed int);
 			break;
 
 		case VIPS_FORMAT_UINT:
-			HCONV(unsigned int, unsigned int);
+			if (conva->max_line < 256)
+				HCONV(unsigned int, unsigned short);
+			else
+				HCONV(unsigned int, unsigned int);
 			break;
 
 		case VIPS_FORMAT_INT:
-			HCONV(signed int, signed int);
+			if (conva->max_line < 256)
+				HCONV(signed int, signed short);
+			else
+				HCONV(signed int, signed int);
 			break;
 
 		case VIPS_FORMAT_FLOAT:
-			HCONV(float, float);
-			break;
-
-		case VIPS_FORMAT_DOUBLE:
-			HCONV(double, double);
-			break;
-
 		case VIPS_FORMAT_COMPLEX:
 			HCONV(float, float);
 			break;
 
+		case VIPS_FORMAT_DOUBLE:
 		case VIPS_FORMAT_DPCOMPLEX:
 			HCONV(double, double);
 			break;
@@ -993,7 +999,7 @@ vips_conva_horizontal(VipsConva *conva, VipsImage *in, VipsImage **out)
 	}
 	(*out)->Bands *= conva->n_hline;
 
-	/* Short u?char lines can use u?short intermediate.
+	/* Short {u,}{char,short,int} lines can use {u,}short as intermediate.
 	 */
 	if (vips_band_format_isuint(in->BandFmt))
 		(*out)->BandFmt = conva->max_line < 256
@@ -1175,34 +1181,41 @@ vips_conva_vgenerate(VipsRegion *out_region,
 		break;
 
 	case VIPS_FORMAT_USHORT:
-		VCONV(unsigned int,
-			unsigned int, unsigned short, CLIP_USHORT);
+		if (conva->max_line < 256)
+			VCONV(unsigned int,
+				unsigned short, unsigned short, CLIP_USHORT);
+		else
+			VCONV(unsigned int,
+				unsigned int, unsigned short, CLIP_USHORT);
 		break;
 
 	case VIPS_FORMAT_SHORT:
-		VCONV(signed int, signed int, signed short, CLIP_SHORT);
+		if (conva->max_line < 256)
+			VCONV(signed int, signed short, signed short, CLIP_SHORT);
+		else
+			VCONV(signed int, signed int, signed short, CLIP_SHORT);
 		break;
 
 	case VIPS_FORMAT_UINT:
-		VCONV(unsigned int, unsigned int, unsigned int, CLIP_NONE);
+		if (conva->max_line < 256)
+			VCONV(unsigned int, unsigned short, unsigned int, CLIP_SHORT);
+		else
+			VCONV(unsigned int, unsigned int, unsigned int, CLIP_NONE);
 		break;
 
 	case VIPS_FORMAT_INT:
-		VCONV(signed int, signed int, signed int, CLIP_NONE);
+		if (conva->max_line < 256)
+			VCONV(signed int, signed short, signed int, CLIP_NONE);
+		else
+			VCONV(signed int, signed int, signed int, CLIP_NONE);
 		break;
 
 	case VIPS_FORMAT_FLOAT:
-		VCONV(float, float, float, CLIP_NONE);
-		break;
-
-	case VIPS_FORMAT_DOUBLE:
-		VCONV(double, double, double, CLIP_NONE);
-		break;
-
 	case VIPS_FORMAT_COMPLEX:
 		VCONV(float, float, float, CLIP_NONE);
 		break;
 
+	case VIPS_FORMAT_DOUBLE:
 	case VIPS_FORMAT_DPCOMPLEX:
 		VCONV(double, double, double, CLIP_NONE);
 		break;
