8323390: Enhance mask blit functionality

Reviewed-by: mbalao
Backport-of: 895893f2fbf1c521d4a263d505f0ecbda8d2eeea

Author: Alexei Voitylov

src/java.desktop/share/classes/sun/java2d/SunGraphics2D.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1996, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1996, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -2151,27 +2151,33 @@ private void doCopyArea(int x, int y, int w, int h, int dx, int dy) {
         }
 
         Blit ob = lastCAblit;
-        if (dy == 0 && dx > 0 && dx < w) {
-            while (w > 0) {
-                int partW = Math.min(w, dx);
-                w -= partW;
-                int sx = x + w;
-                ob.Blit(theData, theData, comp, clip,
-                        sx, y, sx+dx, y+dy, partW, h);
+        try {
+            if (dy == 0 && dx > 0 && dx < w) {
+                while (w > 0) {
+                    int partW = Math.min(w, dx);
+                    w -= partW;
+                    int sx = Math.addExact(x, w);
+                    ob.Blit(theData, theData, comp, clip,
+                            sx, y, sx+dx, y+dy, partW, h);
+                }
+                return;
             }
-            return;
-        }
-        if (dy > 0 && dy < h && dx > -w && dx < w) {
-            while (h > 0) {
-                int partH = Math.min(h, dy);
-                h -= partH;
-                int sy = y + h;
-                ob.Blit(theData, theData, comp, clip,
-                        x, sy, x+dx, sy+dy, w, partH);
+            if (dy > 0 && dy < h && dx > -w && dx < w) {
+                while (h > 0) {
+                    int partH = Math.min(h, dy);
+                    h -= partH;
+                    int sy = Math.addExact(y, h);
+                    ob.Blit(theData, theData, comp, clip,
+                            x, sy, Math.addExact(x, dx), sy+dy, w, partH);
+                }
+                return;
             }
+            ob.Blit(theData, theData, comp, clip, x, y,
+                Math.addExact(x, dx), Math.addExact(y, dy), w, h);
+        } catch (ArithmeticException ex) {
+            // We are hitting integer overflow in Math.addExact()
             return;
         }
-            ob.Blit(theData, theData, comp, clip, x, y, x+dx, y+dy, w, h);
     }
 
     /*

src/java.desktop/share/native/libawt/java2d/SurfaceData.h

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 1999, 2018, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 1999, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -32,6 +32,7 @@
 #define _Included_SurfaceData
 
 #include <jni.h>
+#include <limits.h>
 
 #ifdef __cplusplus
 extern "C" {
@@ -53,6 +54,14 @@ typedef struct {
 
 #define SD_RASINFO_PRIVATE_SIZE         64
 
+#define UNSAFE_TO_ADD(a, b) \
+    (((a >= 0) && (b >= 0) && (a > (INT_MAX - b))) || \
+     ((a < 0) && (b < 0) && (a < (INT_MIN - b)))) \
+
+#define UNSAFE_TO_SUB(a, b) \
+    (((b >= 0) && (a < 0) && (a < (INT_MIN + b))) || \
+     ((b < 0) && (a >= 0) && (-b > (INT_MAX - a)))) \
+
 /*
  * The SurfaceDataRasInfo structure is used to pass in and return various
  * pieces of information about the destination drawable.  In particular:

src/java.desktop/share/native/libawt/java2d/loops/MaskBlit.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2000, 2013, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2000, 2024, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -68,14 +68,28 @@ Java_sun_java2d_loops_MaskBlit_MaskBlit
         return;
     }
 
+    if (width <= 0 || height <= 0) {
+        return;
+    }
+
     srcInfo.bounds.x1 = srcx;
     srcInfo.bounds.y1 = srcy;
+    if (UNSAFE_TO_ADD(srcx, width) ||
+        UNSAFE_TO_ADD(srcy, height) ||
+        UNSAFE_TO_ADD(dstx, width) ||
+        UNSAFE_TO_ADD(dsty, height)) {
+        return;
+    }
     srcInfo.bounds.x2 = srcx + width;
     srcInfo.bounds.y2 = srcy + height;
     dstInfo.bounds.x1 = dstx;
     dstInfo.bounds.y1 = dsty;
     dstInfo.bounds.x2 = dstx + width;
     dstInfo.bounds.y2 = dsty + height;
+    if (UNSAFE_TO_SUB(srcx, dstx) ||
+        UNSAFE_TO_SUB(srcy, dsty)) {
+        return;
+    }
     srcx -= dstx;
     srcy -= dsty;
     SurfaceData_IntersectBounds(&dstInfo.bounds, &clipInfo.bounds);