[GR-18163] Fix capacity computation for huge Hash

PullRequest: truffleruby/3300

Author: eregon

CHANGELOG.md

@@ -8,6 +8,7 @@ Bug fixes:
 * Fix `rb_id2name` to ensure the native string will have the same lifetime as the id (#2630, @aardvark179).
 * Fix `MatchData#[]` exception when passing a length argument larger than the number of match values (#2636, @nirvdrum).
 * Fix `MatchData#[]` exception when supplying a large negative index along with a length argument (@nirvdrum).
+* Fix capacity computation for huge `Hash` (#2635, @eregon).
 
 Compatibility:
 

src/main/java/org/truffleruby/core/hash/library/BucketsHashStore.java

@@ -104,18 +104,30 @@ public BucketsHashStore(Entry[] entries, Entry firstInSequence, Entry lastInSequ
             1073741824 + 85
     };
 
+    private static final int MAX_ARRAY_SIZE = Integer.MAX_VALUE - 8;
+    private static final int MAX_ENTRIES = (int) (MAX_ARRAY_SIZE * LOAD_FACTOR);
+
     // endregion
     // region Utilities
 
     @TruffleBoundary
-    static int capacityGreaterThan(int size) {
+    static int growthCapacityGreaterThan(int size) {
+        int buckets = 0;
         for (int capacity : CAPACITIES) {
             if (capacity > size) {
-                return capacity;
+                buckets = capacity * OVERALLOCATE_FACTOR;
+                break;
             }
         }
 
-        return CAPACITIES[CAPACITIES.length - 1];
+        if (buckets > 0) {
+            assert buckets * LOAD_FACTOR > size;
+            return buckets;
+        } else if (size < MAX_ENTRIES) {
+            return MAX_ARRAY_SIZE;
+        } else {
+            throw new OutOfMemoryError("too big Hash: " + size + " entries");
+        }
     }
 
     static int getBucketIndex(int hashed, int bucketsCount) {
@@ -124,7 +136,7 @@ static int getBucketIndex(int hashed, int bucketsCount) {
 
     @TruffleBoundary
     private void resize(RubyHash hash, int size) {
-        final int bucketsCount = capacityGreaterThan(size) * OVERALLOCATE_FACTOR;
+        final int bucketsCount = growthCapacityGreaterThan(size);
         final Entry[] newEntries = new Entry[bucketsCount];
 
         final Entry firstInSequence = this.firstInSequence;
@@ -589,7 +601,7 @@ public static class GenericHashLiteralNode extends HashLiteralNode {
 
         public GenericHashLiteralNode(RubyNode[] keyValues) {
             super(keyValues);
-            bucketsCount = capacityGreaterThan(keyValues.length / 2) * OVERALLOCATE_FACTOR;
+            bucketsCount = growthCapacityGreaterThan(keyValues.length / 2);
         }
 
         @ExplodeLoop

src/main/java/org/truffleruby/core/hash/library/PackedHashStoreLibrary.java

@@ -124,7 +124,7 @@ private static boolean verifyIntegerHashes(Object[] store) {
 
     @TruffleBoundary
     private static void promoteToBuckets(RubyHash hash, Object[] store, int size) {
-        final Entry[] buckets = new Entry[BucketsHashStore.capacityGreaterThan(size)];
+        final Entry[] buckets = new Entry[BucketsHashStore.growthCapacityGreaterThan(size)];
 
         Entry firstInSequence = null;
         Entry previousInSequence = null;