Fix GH-19790: integer overflowq in imagerectangle

Author: devnexen

ext/gd/libgd/gd.c

@@ -712,13 +712,17 @@ void gdImagePaletteCopy (gdImagePtr to, gdImagePtr from)
  */
 
 static int clip_1d(int *x0, int *y0, int *x1, int *y1, int maxdim) {
-	double m;      /* gradient of line */
+	double m, tmp;      /* gradient of line */
 
 	if (*x0 < 0) {  /* start of line is left of window */
 		if(*x1 < 0) { /* as is the end, so the line never cuts the window */
 			return 0;
 		}
-		m = (*y1 - *y0)/(double)(*x1 - *x0); /* calculate the slope of the line */
+		tmp = (double)*x1 - (double)*x0;
+		if (tmp < (double)INT_MIN || tmp == 0.0 || tmp > (double)INT_MAX) {
+			return 0;
+		}
+		m = (*y1 - *y0)/tmp; /* calculate the slope of the line */
 		/* adjust x0 to be on the left boundary (ie to be zero), and y0 to match */
 		*y0 -= (int)(m * *x0);
 		*x0 = 0;
@@ -733,7 +737,11 @@ static int clip_1d(int *x0, int *y0, int *x1, int *y1, int maxdim) {
 		if (*x1 > maxdim) { /* as is the end, so the line misses the window */
 			return 0;
 		}
-		m = (*y1 - *y0)/(double)(*x1 - *x0);  /* calculate the slope of the line */
+		tmp = (double)*x1 - (double)*x0;
+		if (tmp < (double)INT_MIN || tmp == 0.0 || tmp > (double)INT_MAX) {
+			return 0;
+		}
+		m = (*y1 - *y0)/tmp;  /* calculate the slope of the line */
 		*y0 += (int)(m * (maxdim - *x0)); /* adjust so point is on the right boundary */
 		*x0 = maxdim;
 		/* now, perhaps, adjust the end of the line */
@@ -745,13 +753,21 @@ static int clip_1d(int *x0, int *y0, int *x1, int *y1, int maxdim) {
 	}
 	/* the final case - the start of the line is inside the window */
 	if (*x1 > maxdim) { /* other end is outside to the right */
-		m = (*y1 - *y0)/(double)(*x1 - *x0);  /* calculate the slope of the line */
+		tmp = (double)*x1 - (double)*x0;
+		if (tmp < (double)INT_MIN || tmp == 0.0 || tmp > (double)INT_MAX) {
+			return 0;
+		}
+		m = (*y1 - *y0)/tmp;  /* calculate the slope of the line */
 		*y1 += (int)(m * (maxdim - *x1));
 		*x1 = maxdim;
 		return 1;
 	}
 	if (*x1 < 0) { /* other end is outside to the left */
-		m = (*y1 - *y0)/(double)(*x1 - *x0);  /* calculate the slope of the line */
+		tmp = (double)*x1 - (double)*x0;
+		if (tmp < (double)INT_MIN || tmp == 0.0 || tmp > (double)INT_MAX) {
+			return 0;
+		}
+		m = (*y1 - *y0)/tmp;  /* calculate the slope of the line */
 		*y1 -= (int)(m * *x1);
 		*x1 = 0;
 		return 1;