[GR-23205] Disambiguate out of memory and overflow conditions when resizing sequences

Author: timfel

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/bytes/BytesBuiltins.java

@@ -387,7 +387,7 @@ public abstract static class AddNode extends PythonBinaryBuiltinNode {
 
         @Specialization
         public PBytesLike add(PBytesLike self, PBytesLike other,
-                        @Cached SequenceStorageNodes.ConcatNode concatNode,
+                        @Cached("createWithOverflowError()") SequenceStorageNodes.ConcatNode concatNode,
                         @Cached BytesNodes.CreateBytesNode create) {
             SequenceStorage res = concatNode.execute(self.getSequenceStorage(), other.getSequenceStorage());
             return create.execute(factory(), self, res);
@@ -396,7 +396,7 @@ public PBytesLike add(PBytesLike self, PBytesLike other,
         @Specialization(guards = "bufferLib.isBuffer(buffer)", limit = "3")
         public PBytesLike add(PBytesLike self, Object buffer,
                         @CachedLibrary("buffer") PythonObjectLibrary bufferLib,
-                        @Cached("create()") SequenceStorageNodes.ConcatNode concatNode,
+                        @Cached("createWithOverflowError()") SequenceStorageNodes.ConcatNode concatNode,
                         @Cached BytesNodes.CreateBytesNode create) {
             try {
                 byte[] bytes = bufferLib.getBufferBytes(buffer);
@@ -419,7 +419,7 @@ public Object add(Object self, Object other) {
     public abstract static class MulNode extends PythonBinaryBuiltinNode {
         @Specialization
         public PBytesLike mul(VirtualFrame frame, PBytesLike self, int times,
-                        @Cached("create()") SequenceStorageNodes.RepeatNode repeatNode,
+                        @Cached("createWithOverflowError()") SequenceStorageNodes.RepeatNode repeatNode,
                         @Cached BytesNodes.CreateBytesNode create) {
             SequenceStorage res = repeatNode.execute(frame, self.getSequenceStorage(), times);
             return create.execute(factory(), self, res);
@@ -428,7 +428,7 @@ public PBytesLike mul(VirtualFrame frame, PBytesLike self, int times,
         @Specialization(limit = "getCallSiteInlineCacheMaxDepth()")
         public PBytesLike mul(VirtualFrame frame, PBytesLike self, Object times,
                         @Cached("createBinaryProfile()") ConditionProfile hasFrame,
-                        @Cached("create()") SequenceStorageNodes.RepeatNode repeatNode,
+                        @Cached("createWithOverflowError()") SequenceStorageNodes.RepeatNode repeatNode,
                         @Cached BytesNodes.CreateBytesNode create,
                         @CachedLibrary("times") PythonObjectLibrary lib) {
             SequenceStorage res = repeatNode.execute(frame, self.getSequenceStorage(), getTimesInt(frame, times, hasFrame, lib));

graalpython/com.oracle.graal.python/src/com/oracle/graal/python/builtins/objects/common/SequenceStorageNodes.java

@@ -57,6 +57,7 @@
 
 import com.oracle.graal.python.PythonLanguage;
 import com.oracle.graal.python.builtins.PythonBuiltinClassType;
+import com.oracle.graal.python.builtins.modules.SysModuleBuiltins;
 import com.oracle.graal.python.builtins.objects.cext.capi.CExtNodes;
 import com.oracle.graal.python.builtins.objects.cext.capi.CExtNodes.PCallCapiFunction;
 import com.oracle.graal.python.builtins.objects.cext.capi.CExtNodes.ToSulongNode;
@@ -2333,26 +2334,44 @@ public abstract static class ConcatNode extends SequenceStorageBaseNode {
 
         private final Supplier<GeneralizationNode> genNodeProvider;
 
-        ConcatNode(Supplier<GeneralizationNode> genNodeProvider) {
+        /*
+         * CPython is inconsistent when too repeats are done. Most types raise MemoryError, but
+         * e.g. bytes raises OverflowError when the memory might be available but the size overflows
+         * sys.maxint
+         */
+        private final PythonBuiltinClassType errorForOverflow;
+
+        ConcatNode(Supplier<GeneralizationNode> genNodeProvider, PythonBuiltinClassType errorForOverflow) {
             this.genNodeProvider = genNodeProvider;
+            this.errorForOverflow = errorForOverflow;
         }
 
         public abstract SequenceStorage execute(SequenceStorage left, SequenceStorage right);
 
         @Specialization
         SequenceStorage doRight(SequenceStorage left, SequenceStorage right,
+                        @Cached ConditionProfile shouldOverflow,
                         @Cached PRaiseNode raiseNode,
                         @Cached LenNode lenNode,
                         @Cached BranchProfile outOfMemProfile) {
+            int destlen = 0;
             try {
                 int len1 = lenNode.execute(left);
                 int len2 = lenNode.execute(right);
                 // we eagerly generalize the store to avoid possible cascading generalizations
-                SequenceStorage generalized = generalizeStore(createEmpty(left, right, Math.addExact(len1, len2)), right);
+                destlen = PythonUtils.addExact(len1, len2);
+                if (errorForOverflow == OverflowError && shouldOverflow.profile(destlen >= SysModuleBuiltins.MAXSIZE)) {
+                    // cpython raises an overflow error when this happens
+                    throw raiseNode.raise(OverflowError);
+                }
+                SequenceStorage generalized = generalizeStore(createEmpty(left, right, destlen), right);
                 return doConcat(generalized, left, right);
-            } catch (ArithmeticException | OutOfMemoryError e) {
+            } catch (OutOfMemoryError e) {
                 outOfMemProfile.enter();
                 throw raiseNode.raise(MemoryError);
+            } catch (OverflowException e) {
+                outOfMemProfile.enter();
+                throw raiseNode.raise(errorForOverflow);
             }
         }
 
@@ -2381,15 +2400,23 @@ private SequenceStorage generalizeStore(SequenceStorage storage, Object value) {
         }
 
         public static ConcatNode create() {
-            return ConcatNodeGen.create(() -> NoGeneralizationCustomMessageNode.create(DEFAULT_ERROR_MSG));
+            return create(() -> NoGeneralizationCustomMessageNode.create(DEFAULT_ERROR_MSG), MemoryError);
+        }
+
+        public static ConcatNode createWithOverflowError() {
+            return create(() -> NoGeneralizationCustomMessageNode.create(DEFAULT_ERROR_MSG), OverflowError);
         }
 
         public static ConcatNode create(String msg) {
-            return ConcatNodeGen.create(() -> NoGeneralizationCustomMessageNode.create(msg));
+            return create(() -> NoGeneralizationCustomMessageNode.create(msg), MemoryError);
         }
 
         public static ConcatNode create(Supplier<GeneralizationNode> genNodeProvider) {
-            return ConcatNodeGen.create(genNodeProvider);
+            return create(genNodeProvider, MemoryError);
+        }
+
+        private static ConcatNode create(Supplier<GeneralizationNode> genNodeProvider, PythonBuiltinClassType errorForOverflow) {
+            return ConcatNodeGen.create(genNodeProvider, errorForOverflow);
         }
     }
 
@@ -2491,6 +2518,17 @@ public abstract static class RepeatNode extends SequenceStorageBaseNode {
         @Child private GetItemScalarNode getItemNode;
         @Child private RepeatNode recursive;
 
+        /*
+         * CPython is inconsistent when too repeats are done. Most types raise MemoryError, but
+         * e.g. bytes raises OverflowError when the memory might be available but the size overflows
+         * sys.maxint
+         */
+        private final PythonBuiltinClassType errorForOverflow;
+
+        protected RepeatNode(PythonBuiltinClassType errorForOverflow) {
+            this.errorForOverflow = errorForOverflow;
+        }
+
         public abstract SequenceStorage execute(VirtualFrame frame, SequenceStorage left, Object times);
 
         public abstract SequenceStorage execute(VirtualFrame frame, SequenceStorage left, int times);
@@ -2508,127 +2546,151 @@ static SequenceStorage doZeroRepeat(SequenceStorage s, @SuppressWarnings("unused
 
         /* special but common case: something like '[False] * n' */
         @Specialization(guards = {"s.length() == 1", "times > 0"})
-        static BoolSequenceStorage doBoolSingleElement(BoolSequenceStorage s, int times,
+        BoolSequenceStorage doBoolSingleElement(BoolSequenceStorage s, int times,
                         @Shared("raiseNode") @Cached PRaiseNode raiseNode,
                         @Cached BranchProfile outOfMemProfile) {
             try {
-                boolean[] repeated = new boolean[Math.multiplyExact(s.length(), times)];
+                boolean[] repeated = new boolean[PythonUtils.multiplyExact(s.length(), times)];
                 Arrays.fill(repeated, s.getBoolItemNormalized(0));
                 return new BoolSequenceStorage(repeated);
-            } catch (OutOfMemoryError | ArithmeticException e) {
+            } catch (OutOfMemoryError e) {
                 outOfMemProfile.enter();
                 throw raiseNode.raise(MemoryError);
+            } catch (OverflowException e) {
+                outOfMemProfile.enter();
+                throw raiseNode.raise(errorForOverflow);
             }
         }
 
         /* special but common case: something like '["\x00"] * n' */
         @Specialization(guards = {"s.length() == 1", "times > 0"})
-        static ByteSequenceStorage doByteSingleElement(ByteSequenceStorage s, int times,
+        ByteSequenceStorage doByteSingleElement(ByteSequenceStorage s, int times,
                         @Shared("raiseNode") @Cached PRaiseNode raiseNode,
                         @Cached BranchProfile outOfMemProfile) {
             try {
-                byte[] repeated = new byte[Math.multiplyExact(s.length(), times)];
+                byte[] repeated = new byte[PythonUtils.multiplyExact(s.length(), times)];
                 Arrays.fill(repeated, s.getByteItemNormalized(0));
                 return new ByteSequenceStorage(repeated);
-            } catch (OutOfMemoryError | ArithmeticException e) {
+            } catch (OutOfMemoryError e) {
                 outOfMemProfile.enter();
                 throw raiseNode.raise(MemoryError);
+            } catch (OverflowException e) {
+                outOfMemProfile.enter();
+                throw raiseNode.raise(errorForOverflow);
             }
         }
 
         /* special but common case: something like '["0"] * n' */
         @Specialization(guards = {"s.length() == 1", "times > 0"})
-        static CharSequenceStorage doCharSingleElement(CharSequenceStorage s, int times,
+        CharSequenceStorage doCharSingleElement(CharSequenceStorage s, int times,
                         @Shared("raiseNode") @Cached PRaiseNode raiseNode,
                         @Cached BranchProfile outOfMemProfile) {
             try {
-                char[] repeated = new char[Math.multiplyExact(s.length(), times)];
+                char[] repeated = new char[PythonUtils.multiplyExact(s.length(), times)];
                 Arrays.fill(repeated, s.getCharItemNormalized(0));
                 return new CharSequenceStorage(repeated);
-            } catch (OutOfMemoryError | ArithmeticException e) {
+            } catch (OutOfMemoryError e) {
                 outOfMemProfile.enter();
                 throw raiseNode.raise(MemoryError);
+            } catch (OverflowException e) {
+                outOfMemProfile.enter();
+                throw raiseNode.raise(errorForOverflow);
             }
         }
 
         /* special but common case: something like '[0] * n' */
         @Specialization(guards = {"s.length() == 1", "times > 0"})
-        static IntSequenceStorage doIntSingleElement(IntSequenceStorage s, int times,
+        IntSequenceStorage doIntSingleElement(IntSequenceStorage s, int times,
                         @Shared("raiseNode") @Cached PRaiseNode raiseNode,
                         @Cached BranchProfile outOfMemProfile) {
             try {
-                int[] repeated = new int[Math.multiplyExact(s.length(), times)];
+                int[] repeated = new int[PythonUtils.multiplyExact(s.length(), times)];
                 Arrays.fill(repeated, s.getIntItemNormalized(0));
                 return new IntSequenceStorage(repeated);
-            } catch (OutOfMemoryError | ArithmeticException e) {
+            } catch (OutOfMemoryError e) {
                 outOfMemProfile.enter();
                 throw raiseNode.raise(MemoryError);
+            } catch (OverflowException e) {
+                outOfMemProfile.enter();
+                throw raiseNode.raise(errorForOverflow);
             }
         }
 
         /* special but common case: something like '[0L] * n' */
         @Specialization(guards = {"s.length() == 1", "times > 0"})
-        static LongSequenceStorage doLongSingleElement(LongSequenceStorage s, int times,
+        LongSequenceStorage doLongSingleElement(LongSequenceStorage s, int times,
                         @Shared("raiseNode") @Cached PRaiseNode raiseNode,
                         @Cached BranchProfile outOfMemProfile) {
             try {
-                long[] repeated = new long[Math.multiplyExact(s.length(), times)];
+                long[] repeated = new long[PythonUtils.multiplyExact(s.length(), times)];
                 Arrays.fill(repeated, s.getLongItemNormalized(0));
                 return new LongSequenceStorage(repeated);
-            } catch (OutOfMemoryError | ArithmeticException e) {
+            } catch (OutOfMemoryError e) {
                 outOfMemProfile.enter();
                 throw raiseNode.raise(MemoryError);
+            } catch (OverflowException e) {
+                outOfMemProfile.enter();
+                throw raiseNode.raise(errorForOverflow);
             }
         }
 
         /* special but common case: something like '[0.0] * n' */
         @Specialization(guards = {"s.length() == 1", "times > 0"})
-        static DoubleSequenceStorage doDoubleSingleElement(DoubleSequenceStorage s, int times,
+        DoubleSequenceStorage doDoubleSingleElement(DoubleSequenceStorage s, int times,
                         @Shared("raiseNode") @Cached PRaiseNode raiseNode,
                         @Cached BranchProfile outOfMemProfile) {
             try {
-                double[] repeated = new double[Math.multiplyExact(s.length(), times)];
+                double[] repeated = new double[PythonUtils.multiplyExact(s.length(), times)];
                 Arrays.fill(repeated, s.getDoubleItemNormalized(0));
                 return new DoubleSequenceStorage(repeated);
-            } catch (OutOfMemoryError | ArithmeticException e) {
+            } catch (OutOfMemoryError e) {
                 outOfMemProfile.enter();
                 throw raiseNode.raise(MemoryError);
+            } catch (OverflowException e) {
+                outOfMemProfile.enter();
+                throw raiseNode.raise(errorForOverflow);
             }
         }
 
         /* special but common case: something like '[None] * n' */
         @Specialization(guards = {"s.length() == 1", "times > 0"})
-        static ObjectSequenceStorage doObjectSingleElement(ObjectSequenceStorage s, int times,
+        ObjectSequenceStorage doObjectSingleElement(ObjectSequenceStorage s, int times,
                         @Shared("raiseNode") @Cached PRaiseNode raiseNode,
                         @Cached BranchProfile outOfMemProfile) {
             try {
-                Object[] repeated = new Object[Math.multiplyExact(s.length(), times)];
+                Object[] repeated = new Object[PythonUtils.multiplyExact(s.length(), times)];
                 Arrays.fill(repeated, s.getItemNormalized(0));
                 return new ObjectSequenceStorage(repeated);
-            } catch (OutOfMemoryError | ArithmeticException e) {
+            } catch (OutOfMemoryError e) {
                 outOfMemProfile.enter();
                 throw raiseNode.raise(MemoryError);
+            } catch (OverflowException e) {
+                outOfMemProfile.enter();
+                throw raiseNode.raise(errorForOverflow);
             }
         }
 
         @Specialization(limit = "MAX_ARRAY_STORAGES", guards = {"times > 0", "!isNative(s)", "s.getClass() == cachedClass"})
-        static SequenceStorage doManaged(BasicSequenceStorage s, int times,
+        SequenceStorage doManaged(BasicSequenceStorage s, int times,
                         @Exclusive @Cached PRaiseNode raiseNode,
                         @Cached("create()") BranchProfile outOfMemProfile,
                         @Cached("s.getClass()") Class<? extends SequenceStorage> cachedClass) {
             try {
                 SequenceStorage profiled = cachedClass.cast(s);
                 Object arr1 = profiled.getInternalArrayObject();
                 int len = profiled.length();
-                int newLength = Math.multiplyExact(len, times);
+                int newLength = PythonUtils.multiplyExact(len, times);
                 SequenceStorage repeated = profiled.createEmpty(newLength);
                 Object destArr = repeated.getInternalArrayObject();
                 repeat(destArr, arr1, len, times);
                 repeated.setNewLength(newLength);
                 return repeated;
-            } catch (OutOfMemoryError | ArithmeticException e) {
+            } catch (OutOfMemoryError e) {
                 outOfMemProfile.enter();
                 throw raiseNode.raise(MemoryError);
+            } catch (OverflowException e) {
+                outOfMemProfile.enter();
+                throw raiseNode.raise(errorForOverflow);
             }
         }
 
@@ -2642,7 +2704,7 @@ SequenceStorage doGeneric(SequenceStorage s, int times,
                         @Cached("create()") LenNode lenNode) {
             try {
                 int len = lenNode.execute(s);
-                int newLen = Math.multiplyExact(len, times);
+                int newLen = PythonUtils.multiplyExact(len, times);
                 SequenceStorage repeated = createEmptyNode.execute(s, newLen, -1);
 
                 for (int i = 0; i < len; i++) {
@@ -2658,9 +2720,12 @@ SequenceStorage doGeneric(SequenceStorage s, int times,
 
                 repeated.setNewLength(newLen);
                 return repeated;
-            } catch (OutOfMemoryError | ArithmeticException e) {
+            } catch (OutOfMemoryError e) {
                 outOfMemProfile.enter();
                 throw raiseNode.raise(MemoryError);
+            } catch (OverflowException e) {
+                outOfMemProfile.enter();
+                throw raiseNode.raise(errorForOverflow);
             }
         }
 
@@ -2671,7 +2736,7 @@ SequenceStorage doNonInt(VirtualFrame frame, SequenceStorage s, Object times,
             int i = toIndex(frame, times, raiseNode, lib);
             if (recursive == null) {
                 CompilerDirectives.transferToInterpreterAndInvalidate();
-                recursive = insert(RepeatNodeGen.create());
+                recursive = insert(RepeatNodeGen.create(errorForOverflow));
             }
             return recursive.execute(frame, s, i);
         }
@@ -2702,7 +2767,11 @@ private static int toIndex(VirtualFrame frame, Object times, PRaiseNode raiseNod
         }
 
         public static RepeatNode create() {
-            return RepeatNodeGen.create();
+            return RepeatNodeGen.create(MemoryError);
+        }
+
+        public static RepeatNode createWithOverflowError() {
+            return RepeatNodeGen.create(OverflowError);
         }
     }
 
@@ -3279,9 +3348,9 @@ static BasicSequenceStorage doManaged(BasicSequenceStorage s, int cap,
                 BasicSequenceStorage profiled = cachedClass.cast(s);
                 profiled.ensureCapacity(cap);
                 return profiled;
-            } catch (ArithmeticException | OutOfMemoryError e) {
+            } catch (OutOfMemoryError | ArithmeticException e) {
                 overflowErrorProfile.enter();
-                throw raiseNode.raise(OverflowError);
+                throw raiseNode.raise(MemoryError);
             }
         }